1use crate::{Level, arch_types::ArchTypes, prelude::*, seal::Seal};
7use crate::{
8 f32x4, f32x8, f32x16, f64x2, f64x4, f64x8, i8x16, i8x32, i8x64, i16x8, i16x16, i16x32, i32x4,
9 i32x8, i32x16, i64x2, i64x4, i64x8, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16,
10 mask16x32, mask32x4, mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64,
11 u16x8, u16x16, u16x32, u32x4, u32x8, u32x16, u64x2, u64x4, u64x8,
12};
13#[cfg(target_arch = "x86")]
14use core::arch::x86::*;
15#[cfg(target_arch = "x86_64")]
16use core::arch::x86_64::*;
17use core::ops::*;
18#[doc = "A token for AVX2 intrinsics on `x86` and `x86_64`, representing the x86-64-v3 level."]
19#[doc = "# Browsing the documentation"]
20#[doc = "The method list on this struct is very verbose."]
21#[doc = "A better way to browse the docs is by looking at vector types such as [`u32x4`], [`f32x4`] or [`mask32x4`]."]
22#[doc = "They include all the operations listed here, and also provide some additional convenience methods."]
23#[derive(Clone, Copy, Debug)]
24pub struct Avx2 {
25 _private: (),
26}
27impl Avx2 {
28 #[doc = "Create a SIMD token proving that the x86-64-v3 features are available."]
29 #[doc = r""]
30 #[doc = r" Most users should safely obtain the token from [`Level::new`] instead of calling this function."]
31 #[doc = r""]
32 #[doc = r" This function can be called without an `unsafe` block from a function"]
33 #[doc = r" with all the required target features enabled via the `#[target_feature]` annotation."]
34 #[doc = r""]
35 #[doc = r" # Safety"]
36 #[doc = r""]
37 #[doc = "When invoking this function through an `unsafe` block, the caller must ensure that the current CPU supports `avx2`, `bmi1`, `bmi2`, `cmpxchg16b`, `f16c`, `fma`, `fxsr`, `lzcnt`, `movbe`, `popcnt`, `xsave`."]
38 #[inline]
39 #[target_feature(enable = "avx2,bmi1,bmi2,cmpxchg16b,f16c,fma,fxsr,lzcnt,movbe,popcnt,xsave")]
40 pub const fn assume_supported() -> Self {
41 Self { _private: () }
42 }
43}
44impl Seal for Avx2 {}
45impl ExtractToken for Avx2 {
46 type S = Self;
47 #[inline]
48 fn token(&self) -> Self {
49 *self
50 }
51}
52impl ArchTypes for Avx2 {
53 type f32x4 = crate::support::Aligned128<__m128>;
54 type i8x16 = crate::support::Aligned128<__m128i>;
55 type u8x16 = crate::support::Aligned128<__m128i>;
56 type mask8x16 = crate::support::Aligned128<__m128i>;
57 type i16x8 = crate::support::Aligned128<__m128i>;
58 type u16x8 = crate::support::Aligned128<__m128i>;
59 type mask16x8 = crate::support::Aligned128<__m128i>;
60 type i32x4 = crate::support::Aligned128<__m128i>;
61 type u32x4 = crate::support::Aligned128<__m128i>;
62 type mask32x4 = crate::support::Aligned128<__m128i>;
63 type f64x2 = crate::support::Aligned128<__m128d>;
64 type i64x2 = crate::support::Aligned128<__m128i>;
65 type u64x2 = crate::support::Aligned128<__m128i>;
66 type mask64x2 = crate::support::Aligned128<__m128i>;
67 type f32x8 = crate::support::Aligned256<__m256>;
68 type i8x32 = crate::support::Aligned256<__m256i>;
69 type u8x32 = crate::support::Aligned256<__m256i>;
70 type mask8x32 = crate::support::Aligned256<__m256i>;
71 type i16x16 = crate::support::Aligned256<__m256i>;
72 type u16x16 = crate::support::Aligned256<__m256i>;
73 type mask16x16 = crate::support::Aligned256<__m256i>;
74 type i32x8 = crate::support::Aligned256<__m256i>;
75 type u32x8 = crate::support::Aligned256<__m256i>;
76 type mask32x8 = crate::support::Aligned256<__m256i>;
77 type f64x4 = crate::support::Aligned256<__m256d>;
78 type i64x4 = crate::support::Aligned256<__m256i>;
79 type u64x4 = crate::support::Aligned256<__m256i>;
80 type mask64x4 = crate::support::Aligned256<__m256i>;
81 type f32x16 = crate::support::Aligned512<[__m256; 2usize]>;
82 type i8x64 = crate::support::Aligned512<[__m256i; 2usize]>;
83 type u8x64 = crate::support::Aligned512<[__m256i; 2usize]>;
84 type mask8x64 = crate::support::Aligned512<[__m256i; 2usize]>;
85 type i16x32 = crate::support::Aligned512<[__m256i; 2usize]>;
86 type u16x32 = crate::support::Aligned512<[__m256i; 2usize]>;
87 type mask16x32 = crate::support::Aligned512<[__m256i; 2usize]>;
88 type i32x16 = crate::support::Aligned512<[__m256i; 2usize]>;
89 type u32x16 = crate::support::Aligned512<[__m256i; 2usize]>;
90 type mask32x16 = crate::support::Aligned512<[__m256i; 2usize]>;
91 type f64x8 = crate::support::Aligned512<[__m256d; 2usize]>;
92 type i64x8 = crate::support::Aligned512<[__m256i; 2usize]>;
93 type u64x8 = crate::support::Aligned512<[__m256i; 2usize]>;
94 type mask64x8 = crate::support::Aligned512<[__m256i; 2usize]>;
95}
96impl Simd for Avx2 {
97 type f32s = f32x8<Self>;
98 type f64s = f64x4<Self>;
99 type u8s = u8x32<Self>;
100 type i8s = i8x32<Self>;
101 type u16s = u16x16<Self>;
102 type i16s = i16x16<Self>;
103 type u32s = u32x8<Self>;
104 type i32s = i32x8<Self>;
105 type u64s = u64x4<Self>;
106 type i64s = i64x4<Self>;
107 type mask8s = mask8x32<Self>;
108 type mask16s = mask16x16<Self>;
109 type mask32s = mask32x8<Self>;
110 type mask64s = mask64x4<Self>;
111 #[inline(always)]
112 fn level(self) -> Level {
113 Level::Avx2(self)
114 }
115 #[inline]
116 fn vectorize<F: FnOnce() -> R, R>(self, f: F) -> R {
117 #[inline]
118 #[target_feature(
119 enable = "avx2,bmi1,bmi2,cmpxchg16b,f16c,fma,fxsr,lzcnt,movbe,popcnt,xsave"
120 )]
121 fn vectorize_avx2<F: FnOnce() -> R, R>(f: F) -> R {
122 f()
123 }
124 unsafe { vectorize_avx2(f) }
125 }
126 #[inline(always)]
127 fn abs_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
128 crate::kernel!(
129 #[inline(always)]
130 fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32x4<Avx2> {
131 _mm_andnot_ps(_mm_set1_ps(-0.0), a.into()).simd_into(token)
132 }
133 );
134 kernel(self, a)
135 }
136 #[inline(always)]
137 fn splat_f32x4(self, val: f32) -> f32x4<Self> {
138 crate::kernel!(
139 #[inline(always)]
140 fn kernel(token: Avx2, val: f32) -> f32x4<Avx2> {
141 _mm_set1_ps(val).simd_into(token)
142 }
143 );
144 kernel(self, val)
145 }
146 #[inline(always)]
147 fn reverse_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
148 let indices: u8x16<Self> =
149 [12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3].simd_into(self);
150 self.swizzle_dyn_f32x4(a, indices)
151 }
152 #[inline(always)]
153 fn slide_f32x4<const SHIFT: usize>(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
154 if SHIFT >= 4usize {
155 return b;
156 }
157 let result = dyn_alignr_128(
158 self,
159 Bytes::to_bytes(b).val.0,
160 Bytes::to_bytes(a).val.0,
161 SHIFT * 4usize,
162 );
163 Bytes::from_bytes(u8x16 {
164 val: crate::support::Aligned128(result),
165 simd: self,
166 })
167 }
168 #[inline(always)]
169 fn compress_f32x4(self, values: f32x4<Self>, mask: mask32x4<Self>) -> f32x4<Self> {
170 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
171 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0);
172 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
173 }
174 #[inline(always)]
175 fn compress_merge_f32x4(
176 self,
177 values: f32x4<Self>,
178 mask: mask32x4<Self>,
179 merge: f32x4<Self>,
180 ) -> f32x4<Self> {
181 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
182 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0);
183 let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control);
184 let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0));
185 Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed))
186 }
187 #[inline(always)]
188 fn expand_f32x4(self, values: f32x4<Self>, mask: mask32x4<Self>) -> f32x4<Self> {
189 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
190 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0);
191 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
192 }
193 #[inline(always)]
194 fn expand_merge_f32x4(
195 self,
196 values: f32x4<Self>,
197 mask: mask32x4<Self>,
198 merge: f32x4<Self>,
199 ) -> f32x4<Self> {
200 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
201 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0);
202 let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control));
203 self.select_f32x4(mask, expanded, merge)
204 }
205 #[inline(always)]
206 fn neg_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
207 crate::kernel!(
208 #[inline(always)]
209 fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32x4<Avx2> {
210 _mm_xor_ps(a.into(), _mm_set1_ps(-0.0)).simd_into(token)
211 }
212 );
213 kernel(self, a)
214 }
215 #[inline(always)]
216 fn sqrt_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
217 crate::kernel!(
218 #[inline(always)]
219 fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32x4<Avx2> {
220 _mm_sqrt_ps(a.into()).simd_into(token)
221 }
222 );
223 kernel(self, a)
224 }
225 #[inline(always)]
226 fn approximate_recip_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
227 crate::kernel!(
228 #[inline(always)]
229 fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32x4<Avx2> {
230 _mm_rcp_ps(a.into()).simd_into(token)
231 }
232 );
233 kernel(self, a)
234 }
235 #[inline(always)]
236 fn add_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
237 crate::kernel!(
238 #[inline(always)]
239 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
240 _mm_add_ps(a.into(), b.into()).simd_into(token)
241 }
242 );
243 kernel(self, a, b)
244 }
245 #[inline(always)]
246 fn sub_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
247 crate::kernel!(
248 #[inline(always)]
249 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
250 _mm_sub_ps(a.into(), b.into()).simd_into(token)
251 }
252 );
253 kernel(self, a, b)
254 }
255 #[inline(always)]
256 fn mul_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
257 crate::kernel!(
258 #[inline(always)]
259 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
260 _mm_mul_ps(a.into(), b.into()).simd_into(token)
261 }
262 );
263 kernel(self, a, b)
264 }
265 #[inline(always)]
266 fn div_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
267 crate::kernel!(
268 #[inline(always)]
269 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
270 _mm_div_ps(a.into(), b.into()).simd_into(token)
271 }
272 );
273 kernel(self, a, b)
274 }
275 #[inline(always)]
276 fn copysign_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
277 crate::kernel!(
278 #[inline(always)]
279 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
280 let mask = _mm_set1_ps(-0.0);
281 _mm_or_ps(_mm_and_ps(mask, b.into()), _mm_andnot_ps(mask, a.into()))
282 .simd_into(token)
283 }
284 );
285 kernel(self, a, b)
286 }
287 #[inline(always)]
288 fn reduce_max_f32x4(self, a: f32x4<Self>) -> f32 {
289 crate::kernel!(
290 #[inline(always)]
291 fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32 {
292 let reduced: __m128 = a.into();
293 let shifted = _mm_castsi128_ps(_mm_srli_si128::<8>(_mm_castps_si128(reduced)));
294 let reduced = { _mm_max_ps(reduced, shifted) };
295 let shifted = _mm_castsi128_ps(_mm_srli_si128::<4>(_mm_castps_si128(reduced)));
296 let reduced = { _mm_max_ps(reduced, shifted) };
297 _mm_cvtss_f32(reduced)
298 }
299 );
300 kernel(self, a)
301 }
302 #[inline(always)]
303 fn reduce_min_f32x4(self, a: f32x4<Self>) -> f32 {
304 crate::kernel!(
305 #[inline(always)]
306 fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32 {
307 let reduced: __m128 = a.into();
308 let shifted = _mm_castsi128_ps(_mm_srli_si128::<8>(_mm_castps_si128(reduced)));
309 let reduced = { _mm_min_ps(reduced, shifted) };
310 let shifted = _mm_castsi128_ps(_mm_srli_si128::<4>(_mm_castps_si128(reduced)));
311 let reduced = { _mm_min_ps(reduced, shifted) };
312 _mm_cvtss_f32(reduced)
313 }
314 );
315 kernel(self, a)
316 }
317 #[inline(always)]
318 fn reduce_max_precise_f32x4(self, a: f32x4<Self>) -> f32 {
319 crate::kernel!(
320 #[inline(always)]
321 fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32 {
322 let reduced: __m128 = a.into();
323 let shifted = _mm_castsi128_ps(_mm_srli_si128::<8>(_mm_castps_si128(reduced)));
324 let reduced = {
325 let intermediate = _mm_max_ps(reduced, shifted);
326 let b_is_nan = _mm_cmpunord_ps(shifted, shifted);
327 _mm_blendv_ps(intermediate, reduced, b_is_nan)
328 };
329 let shifted = _mm_castsi128_ps(_mm_srli_si128::<4>(_mm_castps_si128(reduced)));
330 let reduced = {
331 let intermediate = _mm_max_ps(reduced, shifted);
332 let b_is_nan = _mm_cmpunord_ps(shifted, shifted);
333 _mm_blendv_ps(intermediate, reduced, b_is_nan)
334 };
335 _mm_cvtss_f32(reduced)
336 }
337 );
338 kernel(self, a)
339 }
340 #[inline(always)]
341 fn reduce_min_precise_f32x4(self, a: f32x4<Self>) -> f32 {
342 crate::kernel!(
343 #[inline(always)]
344 fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32 {
345 let reduced: __m128 = a.into();
346 let shifted = _mm_castsi128_ps(_mm_srli_si128::<8>(_mm_castps_si128(reduced)));
347 let reduced = {
348 let intermediate = _mm_min_ps(reduced, shifted);
349 let b_is_nan = _mm_cmpunord_ps(shifted, shifted);
350 _mm_blendv_ps(intermediate, reduced, b_is_nan)
351 };
352 let shifted = _mm_castsi128_ps(_mm_srli_si128::<4>(_mm_castps_si128(reduced)));
353 let reduced = {
354 let intermediate = _mm_min_ps(reduced, shifted);
355 let b_is_nan = _mm_cmpunord_ps(shifted, shifted);
356 _mm_blendv_ps(intermediate, reduced, b_is_nan)
357 };
358 _mm_cvtss_f32(reduced)
359 }
360 );
361 kernel(self, a)
362 }
363 #[inline(always)]
364 fn reduce_sum_f32x4(self, a: f32x4<Self>) -> f32 {
365 crate::kernel!(
366 #[inline(always)]
367 fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32 {
368 let a: __m128 = a.into();
369 let adjacent = _mm_add_ps(a, _mm_shuffle_ps::<0b10_11_00_01>(a, a));
370 _mm_cvtss_f32(_mm_add_ss(adjacent, _mm_movehl_ps(adjacent, adjacent)))
371 }
372 );
373 kernel(self, a)
374 }
375 #[inline(always)]
376 fn reduce_product_f32x4(self, a: f32x4<Self>) -> f32 {
377 crate::kernel!(
378 #[inline(always)]
379 fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32 {
380 let a: __m128 = a.into();
381 let adjacent = _mm_mul_ps(a, _mm_shuffle_ps::<0b10_11_00_01>(a, a));
382 _mm_cvtss_f32(_mm_mul_ss(adjacent, _mm_movehl_ps(adjacent, adjacent)))
383 }
384 );
385 kernel(self, a)
386 }
387 #[inline(always)]
388 fn max_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
389 crate::kernel!(
390 #[inline(always)]
391 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
392 _mm_max_ps(a.into(), b.into()).simd_into(token)
393 }
394 );
395 kernel(self, a, b)
396 }
397 #[inline(always)]
398 fn min_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
399 crate::kernel!(
400 #[inline(always)]
401 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
402 _mm_min_ps(a.into(), b.into()).simd_into(token)
403 }
404 );
405 kernel(self, a, b)
406 }
407 #[inline(always)]
408 fn max_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
409 crate::kernel!(
410 #[inline(always)]
411 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
412 let intermediate = _mm_max_ps(a.into(), b.into());
413 let b_is_nan = _mm_cmpunord_ps(b.into(), b.into());
414 _mm_blendv_ps(intermediate, a.into(), b_is_nan).simd_into(token)
415 }
416 );
417 kernel(self, a, b)
418 }
419 #[inline(always)]
420 fn min_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
421 crate::kernel!(
422 #[inline(always)]
423 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
424 let intermediate = _mm_min_ps(a.into(), b.into());
425 let b_is_nan = _mm_cmpunord_ps(b.into(), b.into());
426 _mm_blendv_ps(intermediate, a.into(), b_is_nan).simd_into(token)
427 }
428 );
429 kernel(self, a, b)
430 }
431 #[inline(always)]
432 fn clamp_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
433 crate::kernel!(
434 #[inline(always)]
435 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>, c: f32x4<Avx2>) -> f32x4<Avx2> {
436 debug_assert!(
437 b.simd_le(c).all_true(),
438 "clamp requires min <= max for all elements"
439 );
440 _mm_min_ps(c.into(), _mm_max_ps(b.into(), a.into())).simd_into(token)
441 }
442 );
443 kernel(self, a, b, c)
444 }
445 #[inline(always)]
446 fn simd_eq_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
447 crate::kernel!(
448 #[inline(always)]
449 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> mask32x4<Avx2> {
450 _mm_castps_si128(_mm_cmpeq_ps(a.into(), b.into())).simd_into(token)
451 }
452 );
453 kernel(self, a, b)
454 }
455 #[inline(always)]
456 fn simd_ne_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
457 crate::kernel!(
458 #[inline(always)]
459 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> mask32x4<Avx2> {
460 _mm_castps_si128(_mm_cmpneq_ps(a.into(), b.into())).simd_into(token)
461 }
462 );
463 kernel(self, a, b)
464 }
465 #[inline(always)]
466 fn simd_lt_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
467 crate::kernel!(
468 #[inline(always)]
469 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> mask32x4<Avx2> {
470 _mm_castps_si128(_mm_cmplt_ps(a.into(), b.into())).simd_into(token)
471 }
472 );
473 kernel(self, a, b)
474 }
475 #[inline(always)]
476 fn simd_le_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
477 crate::kernel!(
478 #[inline(always)]
479 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> mask32x4<Avx2> {
480 _mm_castps_si128(_mm_cmple_ps(a.into(), b.into())).simd_into(token)
481 }
482 );
483 kernel(self, a, b)
484 }
485 #[inline(always)]
486 fn zip_low_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
487 crate::kernel!(
488 #[inline(always)]
489 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
490 _mm_unpacklo_ps(a.into(), b.into()).simd_into(token)
491 }
492 );
493 kernel(self, a, b)
494 }
495 #[inline(always)]
496 fn zip_high_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
497 crate::kernel!(
498 #[inline(always)]
499 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
500 _mm_unpackhi_ps(a.into(), b.into()).simd_into(token)
501 }
502 );
503 kernel(self, a, b)
504 }
505 #[inline(always)]
506 fn unzip_low_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
507 crate::kernel!(
508 #[inline(always)]
509 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
510 _mm_shuffle_ps::<0b10_00_10_00>(a.into(), b.into()).simd_into(token)
511 }
512 );
513 kernel(self, a, b)
514 }
515 #[inline(always)]
516 fn unzip_high_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
517 crate::kernel!(
518 #[inline(always)]
519 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
520 _mm_shuffle_ps::<0b11_01_11_01>(a.into(), b.into()).simd_into(token)
521 }
522 );
523 kernel(self, a, b)
524 }
525 #[inline(always)]
526 fn interleave_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> (f32x4<Self>, f32x4<Self>) {
527 (self.zip_low_f32x4(a, b), self.zip_high_f32x4(a, b))
528 }
529 #[inline(always)]
530 fn deinterleave_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> (f32x4<Self>, f32x4<Self>) {
531 (self.unzip_low_f32x4(a, b), self.unzip_high_f32x4(a, b))
532 }
533 #[inline(always)]
534 fn mul_add_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
535 crate::kernel!(
536 #[inline(always)]
537 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>, c: f32x4<Avx2>) -> f32x4<Avx2> {
538 _mm_fmadd_ps(a.into(), b.into(), c.into()).simd_into(token)
539 }
540 );
541 kernel(self, a, b, c)
542 }
543 #[inline(always)]
544 fn mul_add_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
545 self.mul_add_f32x4(a, b, c)
546 }
547 #[inline(always)]
548 fn mul_sub_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
549 crate::kernel!(
550 #[inline(always)]
551 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>, c: f32x4<Avx2>) -> f32x4<Avx2> {
552 _mm_fmsub_ps(a.into(), b.into(), c.into()).simd_into(token)
553 }
554 );
555 kernel(self, a, b, c)
556 }
557 #[inline(always)]
558 fn mul_sub_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
559 self.mul_sub_f32x4(a, b, c)
560 }
561 #[inline(always)]
562 fn floor_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
563 crate::kernel!(
564 #[inline(always)]
565 fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32x4<Avx2> {
566 _mm_round_ps::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
567 .simd_into(token)
568 }
569 );
570 kernel(self, a)
571 }
572 #[inline(always)]
573 fn ceil_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
574 crate::kernel!(
575 #[inline(always)]
576 fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32x4<Avx2> {
577 _mm_round_ps::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
578 .simd_into(token)
579 }
580 );
581 kernel(self, a)
582 }
583 #[inline(always)]
584 fn round_ties_even_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
585 crate::kernel!(
586 #[inline(always)]
587 fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32x4<Avx2> {
588 _mm_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
589 .simd_into(token)
590 }
591 );
592 kernel(self, a)
593 }
594 #[inline(always)]
595 fn round_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
596 let bias = self.splat_f32x4(const { f32::next_down(0.5) });
597 (a + self.copysign_f32x4(bias, a)).trunc()
598 }
599 #[inline(always)]
600 fn fract_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
601 a - self.trunc_f32x4(a)
602 }
603 #[inline(always)]
604 fn trunc_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
605 crate::kernel!(
606 #[inline(always)]
607 fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32x4<Avx2> {
608 _mm_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
609 .simd_into(token)
610 }
611 );
612 kernel(self, a)
613 }
614 #[inline(always)]
615 fn to_degrees_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
616 a * const { f32::to_degrees(1.0) }
617 }
618 #[inline(always)]
619 fn to_radians_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
620 a * const { f32::to_radians(1.0) }
621 }
622 #[inline(always)]
623 fn is_nan_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
624 a.simd_ne(a)
625 }
626 #[inline(always)]
627 fn is_infinite_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
628 a.abs().simd_eq(f32::INFINITY)
629 }
630 #[inline(always)]
631 fn is_finite_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
632 a.abs().simd_lt(f32::INFINITY)
633 }
634 #[inline(always)]
635 fn is_subnormal_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
636 let i: i32x4<Self> = a.bitcast();
637 (i & 0x7F80_0000_i32).simd_eq(0) & (i & 0x007F_FFFF_i32).simd_ne(0)
638 }
639 #[inline(always)]
640 fn is_normal_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
641 let i: i32x4<Self> = a.bitcast();
642 let exp = i & 0x7F80_0000_i32;
643 exp.simd_ne(0) & exp.simd_ne(0x7F80_0000_i32)
644 }
645 #[inline(always)]
646 fn is_sign_positive_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
647 let i: i32x4<Self> = a.bitcast();
648 (i & i32::MIN).simd_eq(0)
649 }
650 #[inline(always)]
651 fn is_sign_negative_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
652 !a.is_sign_positive()
653 }
654 #[inline(always)]
655 fn select_f32x4(self, a: mask32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
656 crate::kernel!(
657 #[inline(always)]
658 fn kernel(
659 token: Avx2,
660 a: mask32x4<Avx2>,
661 b: f32x4<Avx2>,
662 c: f32x4<Avx2>,
663 ) -> f32x4<Avx2> {
664 _mm_blendv_ps(c.into(), b.into(), _mm_castsi128_ps(a.into())).simd_into(token)
665 }
666 );
667 kernel(self, a, b, c)
668 }
669 #[inline(always)]
670 fn combine_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x8<Self> {
671 crate::kernel!(
672 #[inline(always)]
673 fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x8<Avx2> {
674 _mm256_setr_m128(a.into(), b.into()).simd_into(token)
675 }
676 );
677 kernel(self, a, b)
678 }
679 #[inline(always)]
680 fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4<Self>; 4usize] {
681 crate::kernel!(
682 #[inline(always)]
683 fn kernel(token: Avx2, src: &[f32; 16usize]) -> [f32x4<Avx2>; 4usize] {
684 let (chunks, []) = src.as_chunks::<4usize>() else {
685 unreachable!()
686 };
687 let v0: __m128 =
688 crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[0]);
689 let v1: __m128 =
690 crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[1]);
691 let v2: __m128 =
692 crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[2]);
693 let v3: __m128 =
694 crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[3]);
695 let tmp0 = _mm_unpacklo_ps(v0, v1);
696 let tmp1 = _mm_unpackhi_ps(v0, v1);
697 let tmp2 = _mm_unpacklo_ps(v2, v3);
698 let tmp3 = _mm_unpackhi_ps(v2, v3);
699 let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2)));
700 let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2)));
701 let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3)));
702 let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3)));
703 [
704 out0.simd_into(token),
705 out1.simd_into(token),
706 out2.simd_into(token),
707 out3.simd_into(token),
708 ]
709 }
710 );
711 kernel(self, src)
712 }
713 #[inline(always)]
714 fn store_four_interleaved_f32x4(
715 self,
716 vectors: [f32x4<Self>; 4usize],
717 dest: &mut [f32; 16usize],
718 ) -> () {
719 crate::kernel!(
720 #[inline(always)]
721 fn kernel(
722 token: Avx2,
723 vectors: [f32x4<Avx2>; 4usize],
724 dest: &mut [f32; 16usize],
725 ) -> () {
726 let _ = token;
727 let v0: __m128 = vectors[0].into();
728 let v1: __m128 = vectors[1].into();
729 let v2: __m128 = vectors[2].into();
730 let v3: __m128 = vectors[3].into();
731 let tmp0 = _mm_unpacklo_ps(v0, v1);
732 let tmp1 = _mm_unpackhi_ps(v0, v1);
733 let tmp2 = _mm_unpacklo_ps(v2, v3);
734 let tmp3 = _mm_unpackhi_ps(v2, v3);
735 let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2)));
736 let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2)));
737 let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3)));
738 let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3)));
739 let (chunks, []) = dest.as_chunks_mut::<4usize>() else {
740 unreachable!()
741 };
742 crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>(
743 out0,
744 &mut chunks[0],
745 );
746 crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>(
747 out1,
748 &mut chunks[1],
749 );
750 crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>(
751 out2,
752 &mut chunks[2],
753 );
754 crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>(
755 out3,
756 &mut chunks[3],
757 );
758 }
759 );
760 kernel(self, vectors, dest);
761 }
762 #[inline(always)]
763 fn widen_f32x4(self, a: f32x4<Self>) -> (f64x2<Self>, f64x2<Self>) {
764 crate::kernel!(
765 #[inline(always)]
766 fn kernel(token: Avx2, a: f32x4<Avx2>) -> (f64x2<Avx2>, f64x2<Avx2>) {
767 let raw = a.into();
768 (
769 _mm_cvtps_pd(raw).simd_into(token),
770 _mm_cvtps_pd(_mm_movehl_ps(raw, raw)).simd_into(token),
771 )
772 }
773 );
774 kernel(self, a)
775 }
776 #[inline(always)]
777 fn cvt_u32_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
778 crate::kernel!(
779 #[inline(always)]
780 fn kernel(token: Avx2, a: f32x4<Avx2>) -> u32x4<Avx2> {
781 let mut converted = _mm_cvttps_epi32(a.into());
782 let in_range = _mm_cmplt_ps(a.into(), _mm_set1_ps(2147483648.0));
783 let all_in_range = _mm_movemask_ps(in_range) == 0b1111;
784 if !all_in_range {
785 let excess = _mm_sub_ps(a.into(), _mm_set1_ps(2147483648.0));
786 let excess_converted = _mm_cvttps_epi32(_mm_andnot_ps(in_range, excess));
787 converted = _mm_add_epi32(converted, excess_converted);
788 }
789 converted.simd_into(token)
790 }
791 );
792 kernel(self, a)
793 }
794 #[inline(always)]
795 fn cvt_u32_precise_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
796 crate::kernel!(
797 #[inline(always)]
798 fn kernel(token: Avx2, a: f32x4<Avx2>) -> u32x4<Avx2> {
799 let a = _mm_max_ps(a.into(), _mm_setzero_ps());
800 let mut converted = _mm_cvttps_epi32(a);
801 let in_range = _mm_cmplt_ps(a, _mm_set1_ps(2147483648.0));
802 let all_in_range = _mm_movemask_ps(in_range) == 0b1111;
803 if !all_in_range {
804 let exceeds_unsigned_range =
805 _mm_castps_si128(_mm_cmplt_ps(_mm_set1_ps(4294967040.0), a));
806 let excess = _mm_sub_ps(a, _mm_set1_ps(2147483648.0));
807 let excess_converted = _mm_cvttps_epi32(_mm_andnot_ps(in_range, excess));
808 converted = _mm_add_epi32(converted, excess_converted);
809 converted = _mm_or_si128(converted, exceeds_unsigned_range);
810 }
811 converted.simd_into(token)
812 }
813 );
814 kernel(self, a)
815 }
816 #[inline(always)]
817 fn cvt_i32_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
818 crate::kernel!(
819 #[inline(always)]
820 fn kernel(token: Avx2, a: f32x4<Avx2>) -> i32x4<Avx2> {
821 _mm_cvttps_epi32(a.into()).simd_into(token)
822 }
823 );
824 kernel(self, a)
825 }
826 #[inline(always)]
827 fn cvt_i32_precise_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
828 crate::kernel!(
829 #[inline(always)]
830 fn kernel(token: Avx2, a: f32x4<Avx2>) -> i32x4<Avx2> {
831 let a = a.into();
832 let converted = _mm_cvttps_epi32(a);
833 let positive_overflow =
834 _mm_castps_si128(_mm_cmple_ps(_mm_set1_ps(2147483648.0), a));
835 let converted = _mm_xor_si128(converted, positive_overflow);
836 let is_not_nan = _mm_castps_si128(_mm_cmpord_ps(a, a));
837 _mm_and_si128(converted, is_not_nan).simd_into(token)
838 }
839 );
840 kernel(self, a)
841 }
842 #[inline(always)]
843 fn abs_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
844 crate::kernel!(
845 #[inline(always)]
846 fn kernel(token: Avx2, a: i8x16<Avx2>) -> i8x16<Avx2> {
847 _mm_abs_epi8(a.into()).simd_into(token)
848 }
849 );
850 kernel(self, a)
851 }
852 #[inline(always)]
853 fn splat_i8x16(self, val: i8) -> i8x16<Self> {
854 crate::kernel!(
855 #[inline(always)]
856 fn kernel(token: Avx2, val: i8) -> i8x16<Avx2> {
857 _mm_set1_epi8(val).simd_into(token)
858 }
859 );
860 kernel(self, val)
861 }
862 #[inline(always)]
863 fn reverse_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
864 let indices: u8x16<Self> =
865 [15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0].simd_into(self);
866 self.swizzle_dyn_i8x16(a, indices)
867 }
868 #[inline(always)]
869 fn slide_i8x16<const SHIFT: usize>(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
870 if SHIFT >= 16usize {
871 return b;
872 }
873 let result = dyn_alignr_128(
874 self,
875 Bytes::to_bytes(b).val.0,
876 Bytes::to_bytes(a).val.0,
877 SHIFT,
878 );
879 Bytes::from_bytes(u8x16 {
880 val: crate::support::Aligned128(result),
881 simd: self,
882 })
883 }
884 #[inline(always)]
885 fn compress_i8x16(self, values: i8x16<Self>, mask: mask8x16<Self>) -> i8x16<Self> {
886 let mask = mask8x16 {
887 val: crate::transmute::checked_transmute_copy(&mask.val),
888 simd: self,
889 };
890 Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask))
891 }
892 #[inline(always)]
893 fn compress_merge_i8x16(
894 self,
895 values: i8x16<Self>,
896 mask: mask8x16<Self>,
897 merge: i8x16<Self>,
898 ) -> i8x16<Self> {
899 let mask = mask8x16 {
900 val: crate::transmute::checked_transmute_copy(&mask.val),
901 simd: self,
902 };
903 Bytes::from_bytes(self.compress_merge_u8x16(
904 Bytes::to_bytes(values),
905 mask,
906 Bytes::to_bytes(merge),
907 ))
908 }
909 #[inline(always)]
910 fn expand_i8x16(self, values: i8x16<Self>, mask: mask8x16<Self>) -> i8x16<Self> {
911 let mask = mask8x16 {
912 val: crate::transmute::checked_transmute_copy(&mask.val),
913 simd: self,
914 };
915 Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask))
916 }
917 #[inline(always)]
918 fn expand_merge_i8x16(
919 self,
920 values: i8x16<Self>,
921 mask: mask8x16<Self>,
922 merge: i8x16<Self>,
923 ) -> i8x16<Self> {
924 let mask = mask8x16 {
925 val: crate::transmute::checked_transmute_copy(&mask.val),
926 simd: self,
927 };
928 Bytes::from_bytes(self.expand_merge_u8x16(
929 Bytes::to_bytes(values),
930 mask,
931 Bytes::to_bytes(merge),
932 ))
933 }
934 #[inline(always)]
935 fn count_ones_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
936 crate::kernel!(
937 #[inline(always)]
938 fn kernel(token: Avx2, a: i8x16<Avx2>) -> i8x16<Avx2> {
939 let value = a.into();
940 let nibble_mask = _mm_set1_epi8(0x0f);
941 let lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4);
942 let low = _mm_and_si128(value, nibble_mask);
943 let high = _mm_and_si128(_mm_srli_epi16::<4>(value), nibble_mask);
944 let byte_counts = _mm_add_epi8(
945 _mm_shuffle_epi8(lookup, low),
946 _mm_shuffle_epi8(lookup, high),
947 );
948 byte_counts.simd_into(token)
949 }
950 );
951 kernel(self, a)
952 }
953 #[inline(always)]
954 fn count_zeros_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
955 self.count_ones_i8x16(self.not_i8x16(a))
956 }
957 #[inline(always)]
958 fn add_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
959 crate::kernel!(
960 #[inline(always)]
961 fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
962 _mm_add_epi8(a.into(), b.into()).simd_into(token)
963 }
964 );
965 kernel(self, a, b)
966 }
967 #[inline(always)]
968 fn saturating_add_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
969 crate::kernel!(
970 #[inline(always)]
971 fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
972 _mm_adds_epi8(a.into(), b.into()).simd_into(token)
973 }
974 );
975 kernel(self, a, b)
976 }
977 #[inline(always)]
978 fn sub_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
979 crate::kernel!(
980 #[inline(always)]
981 fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
982 _mm_sub_epi8(a.into(), b.into()).simd_into(token)
983 }
984 );
985 kernel(self, a, b)
986 }
987 #[inline(always)]
988 fn saturating_sub_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
989 crate::kernel!(
990 #[inline(always)]
991 fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
992 _mm_subs_epi8(a.into(), b.into()).simd_into(token)
993 }
994 );
995 kernel(self, a, b)
996 }
997 #[inline(always)]
998 fn mul_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
999 crate::kernel!(
1000 #[inline(always)]
1001 fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
1002 let a = a.into();
1003 let b = b.into();
1004 let low_mask = _mm_set1_epi16(0xFF);
1005 let dst_even = _mm_mullo_epi16(a, b);
1006 let dst_odd = _mm_maddubs_epi16(a, _mm_andnot_si128(low_mask, b));
1007 _mm_or_si128(
1008 _mm_slli_epi16(dst_odd, 8),
1009 _mm_and_si128(dst_even, low_mask),
1010 )
1011 .simd_into(token)
1012 }
1013 );
1014 kernel(self, a, b)
1015 }
1016 #[inline(always)]
1017 fn and_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1018 crate::kernel!(
1019 #[inline(always)]
1020 fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
1021 _mm_and_si128(a.into(), b.into()).simd_into(token)
1022 }
1023 );
1024 kernel(self, a, b)
1025 }
1026 #[inline(always)]
1027 fn or_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1028 crate::kernel!(
1029 #[inline(always)]
1030 fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
1031 _mm_or_si128(a.into(), b.into()).simd_into(token)
1032 }
1033 );
1034 kernel(self, a, b)
1035 }
1036 #[inline(always)]
1037 fn xor_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1038 crate::kernel!(
1039 #[inline(always)]
1040 fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
1041 _mm_xor_si128(a.into(), b.into()).simd_into(token)
1042 }
1043 );
1044 kernel(self, a, b)
1045 }
1046 #[inline(always)]
1047 fn not_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
1048 a ^ !0
1049 }
1050 #[inline(always)]
1051 fn shl_i8x16(self, a: i8x16<Self>, shift: u32) -> i8x16<Self> {
1052 crate::kernel!(
1053 #[inline(always)]
1054 fn kernel(token: Avx2, a: i8x16<Avx2>, shift: u32) -> i8x16<Avx2> {
1055 let val = a.into();
1056 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
1057 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
1058 let byte_mask = _mm_set1_epi8(mask_byte);
1059 _mm_sll_epi16(_mm_and_si128(val, byte_mask), shift_count).simd_into(token)
1060 }
1061 );
1062 kernel(self, a, shift)
1063 }
1064 #[inline(always)]
1065 fn shlv_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1066 [
1067 i8::wrapping_shl(a[0usize], b[0usize] as u32),
1068 i8::wrapping_shl(a[1usize], b[1usize] as u32),
1069 i8::wrapping_shl(a[2usize], b[2usize] as u32),
1070 i8::wrapping_shl(a[3usize], b[3usize] as u32),
1071 i8::wrapping_shl(a[4usize], b[4usize] as u32),
1072 i8::wrapping_shl(a[5usize], b[5usize] as u32),
1073 i8::wrapping_shl(a[6usize], b[6usize] as u32),
1074 i8::wrapping_shl(a[7usize], b[7usize] as u32),
1075 i8::wrapping_shl(a[8usize], b[8usize] as u32),
1076 i8::wrapping_shl(a[9usize], b[9usize] as u32),
1077 i8::wrapping_shl(a[10usize], b[10usize] as u32),
1078 i8::wrapping_shl(a[11usize], b[11usize] as u32),
1079 i8::wrapping_shl(a[12usize], b[12usize] as u32),
1080 i8::wrapping_shl(a[13usize], b[13usize] as u32),
1081 i8::wrapping_shl(a[14usize], b[14usize] as u32),
1082 i8::wrapping_shl(a[15usize], b[15usize] as u32),
1083 ]
1084 .simd_into(self)
1085 }
1086 #[inline(always)]
1087 fn shr_i8x16(self, a: i8x16<Self>, shift: u32) -> i8x16<Self> {
1088 crate::kernel!(
1089 #[inline(always)]
1090 fn kernel(token: Avx2, a: i8x16<Avx2>, shift: u32) -> i8x16<Avx2> {
1091 let val = a.into();
1092 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
1093 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
1094 let byte_mask = _mm_set1_epi8(mask_byte);
1095 let shifted = _mm_srl_epi16(val, shift_count);
1096 let result = {
1097 let sign = _mm_cmpgt_epi8(_mm_setzero_si128(), val);
1098 _mm_or_si128(
1099 _mm_and_si128(shifted, byte_mask),
1100 _mm_andnot_si128(byte_mask, sign),
1101 )
1102 };
1103 result.simd_into(token)
1104 }
1105 );
1106 kernel(self, a, shift)
1107 }
1108 #[inline(always)]
1109 fn shrv_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1110 [
1111 i8::wrapping_shr(a[0usize], b[0usize] as u32),
1112 i8::wrapping_shr(a[1usize], b[1usize] as u32),
1113 i8::wrapping_shr(a[2usize], b[2usize] as u32),
1114 i8::wrapping_shr(a[3usize], b[3usize] as u32),
1115 i8::wrapping_shr(a[4usize], b[4usize] as u32),
1116 i8::wrapping_shr(a[5usize], b[5usize] as u32),
1117 i8::wrapping_shr(a[6usize], b[6usize] as u32),
1118 i8::wrapping_shr(a[7usize], b[7usize] as u32),
1119 i8::wrapping_shr(a[8usize], b[8usize] as u32),
1120 i8::wrapping_shr(a[9usize], b[9usize] as u32),
1121 i8::wrapping_shr(a[10usize], b[10usize] as u32),
1122 i8::wrapping_shr(a[11usize], b[11usize] as u32),
1123 i8::wrapping_shr(a[12usize], b[12usize] as u32),
1124 i8::wrapping_shr(a[13usize], b[13usize] as u32),
1125 i8::wrapping_shr(a[14usize], b[14usize] as u32),
1126 i8::wrapping_shr(a[15usize], b[15usize] as u32),
1127 ]
1128 .simd_into(self)
1129 }
1130 #[inline(always)]
1131 fn reduce_max_i8x16(self, a: i8x16<Self>) -> i8 {
1132 crate::kernel!(
1133 #[inline(always)]
1134 fn kernel(token: Avx2, a: i8x16<Avx2>) -> i8 {
1135 let reduced: __m128i = a.into();
1136 let shifted = _mm_srli_si128::<8>(reduced);
1137 let reduced = { _mm_max_epi8(reduced, shifted) };
1138 let shifted = _mm_srli_si128::<4>(reduced);
1139 let reduced = { _mm_max_epi8(reduced, shifted) };
1140 let shifted = _mm_srli_si128::<2>(reduced);
1141 let reduced = { _mm_max_epi8(reduced, shifted) };
1142 let shifted = _mm_srli_si128::<1>(reduced);
1143 let reduced = { _mm_max_epi8(reduced, shifted) };
1144 {
1145 let lanes: [i8; 16usize] = crate::transmute::checked_transmute_copy(&reduced);
1146 lanes[0]
1147 }
1148 }
1149 );
1150 kernel(self, a)
1151 }
1152 #[inline(always)]
1153 fn reduce_min_i8x16(self, a: i8x16<Self>) -> i8 {
1154 crate::kernel!(
1155 #[inline(always)]
1156 fn kernel(token: Avx2, a: i8x16<Avx2>) -> i8 {
1157 let reduced: __m128i = a.into();
1158 let shifted = _mm_srli_si128::<8>(reduced);
1159 let reduced = { _mm_min_epi8(reduced, shifted) };
1160 let shifted = _mm_srli_si128::<4>(reduced);
1161 let reduced = { _mm_min_epi8(reduced, shifted) };
1162 let shifted = _mm_srli_si128::<2>(reduced);
1163 let reduced = { _mm_min_epi8(reduced, shifted) };
1164 let shifted = _mm_srli_si128::<1>(reduced);
1165 let reduced = { _mm_min_epi8(reduced, shifted) };
1166 {
1167 let lanes: [i8; 16usize] = crate::transmute::checked_transmute_copy(&reduced);
1168 lanes[0]
1169 }
1170 }
1171 );
1172 kernel(self, a)
1173 }
1174 #[inline(always)]
1175 fn reduce_sum_i8x16(self, a: i8x16<Self>) -> i8 {
1176 crate::kernel!(
1177 #[inline(always)]
1178 fn kernel(token: Avx2, a: i8x16<Avx2>) -> i8 {
1179 let sum: __m128i = a.into();
1180 let sum = _mm_add_epi8(sum, _mm_srli_si128::<8>(sum));
1181 let sum = _mm_add_epi8(sum, _mm_srli_si128::<4>(sum));
1182 let sum = _mm_add_epi8(sum, _mm_srli_si128::<2>(sum));
1183 let sum = _mm_add_epi8(sum, _mm_srli_si128::<1>(sum));
1184 let lanes: [i8; 16usize] = crate::transmute::checked_transmute_copy(&sum);
1185 lanes[0]
1186 }
1187 );
1188 kernel(self, a)
1189 }
1190 #[inline(always)]
1191 fn reduce_product_i8x16(self, a: i8x16<Self>) -> i8 {
1192 crate::kernel!(
1193 #[inline(always)]
1194 fn kernel(token: Avx2, a: i8x16<Avx2>) -> i8 {
1195 let value: __m128i = a.into();
1196 let product = _mm_mullo_epi16(
1197 _mm_unpacklo_epi8(value, value),
1198 _mm_unpackhi_epi8(value, value),
1199 );
1200 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b11_10_11_10>(product));
1201 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b01_01_01_01>(product));
1202 let product = _mm_mullo_epi16(product, _mm_srli_epi32::<16>(product));
1203 let lanes: [i8; 16usize] = crate::transmute::checked_transmute_copy(&product);
1204 lanes[0]
1205 }
1206 );
1207 kernel(self, a)
1208 }
1209 #[inline(always)]
1210 fn max_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1211 crate::kernel!(
1212 #[inline(always)]
1213 fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
1214 _mm_max_epi8(a.into(), b.into()).simd_into(token)
1215 }
1216 );
1217 kernel(self, a, b)
1218 }
1219 #[inline(always)]
1220 fn min_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1221 crate::kernel!(
1222 #[inline(always)]
1223 fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
1224 _mm_min_epi8(a.into(), b.into()).simd_into(token)
1225 }
1226 );
1227 kernel(self, a, b)
1228 }
1229 #[inline(always)]
1230 fn clamp_i8x16(self, a: i8x16<Self>, b: i8x16<Self>, c: i8x16<Self>) -> i8x16<Self> {
1231 debug_assert!(
1232 b.simd_le(c).all_true(),
1233 "clamp requires min <= max for all elements"
1234 );
1235 a.max(b).min(c)
1236 }
1237 #[inline(always)]
1238 fn simd_eq_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
1239 crate::kernel!(
1240 #[inline(always)]
1241 fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> mask8x16<Avx2> {
1242 _mm_cmpeq_epi8(a.into(), b.into()).simd_into(token)
1243 }
1244 );
1245 kernel(self, a, b)
1246 }
1247 #[inline(always)]
1248 fn simd_lt_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
1249 crate::kernel!(
1250 #[inline(always)]
1251 fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> mask8x16<Avx2> {
1252 _mm_cmpgt_epi8(b.into(), a.into()).simd_into(token)
1253 }
1254 );
1255 kernel(self, a, b)
1256 }
1257 #[inline(always)]
1258 fn simd_le_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
1259 crate::kernel!(
1260 #[inline(always)]
1261 fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> mask8x16<Avx2> {
1262 _mm_cmpeq_epi8(_mm_min_epi8(a.into(), b.into()), a.into()).simd_into(token)
1263 }
1264 );
1265 kernel(self, a, b)
1266 }
1267 #[inline(always)]
1268 fn zip_low_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1269 crate::kernel!(
1270 #[inline(always)]
1271 fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
1272 _mm_unpacklo_epi8(a.into(), b.into()).simd_into(token)
1273 }
1274 );
1275 kernel(self, a, b)
1276 }
1277 #[inline(always)]
1278 fn zip_high_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1279 crate::kernel!(
1280 #[inline(always)]
1281 fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
1282 _mm_unpackhi_epi8(a.into(), b.into()).simd_into(token)
1283 }
1284 );
1285 kernel(self, a, b)
1286 }
1287 #[inline(always)]
1288 fn unzip_low_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1289 crate::kernel!(
1290 #[inline(always)]
1291 fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
1292 let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
1293 let t1 = _mm_shuffle_epi8(a.into(), mask);
1294 let t2 = _mm_shuffle_epi8(b.into(), mask);
1295 _mm_unpacklo_epi64(t1, t2).simd_into(token)
1296 }
1297 );
1298 kernel(self, a, b)
1299 }
1300 #[inline(always)]
1301 fn unzip_high_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1302 crate::kernel!(
1303 #[inline(always)]
1304 fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
1305 let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
1306 let t1 = _mm_shuffle_epi8(a.into(), mask);
1307 let t2 = _mm_shuffle_epi8(b.into(), mask);
1308 _mm_unpackhi_epi64(t1, t2).simd_into(token)
1309 }
1310 );
1311 kernel(self, a, b)
1312 }
1313 #[inline(always)]
1314 fn interleave_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i8x16<Self>, i8x16<Self>) {
1315 (self.zip_low_i8x16(a, b), self.zip_high_i8x16(a, b))
1316 }
1317 #[inline(always)]
1318 fn deinterleave_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i8x16<Self>, i8x16<Self>) {
1319 crate::kernel!(
1320 #[inline(always)]
1321 fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> (i8x16<Avx2>, i8x16<Avx2>) {
1322 let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
1323 let a = _mm_shuffle_epi8(a.into(), mask);
1324 let b = _mm_shuffle_epi8(b.into(), mask);
1325 (
1326 _mm_unpacklo_epi64(a, b).simd_into(token),
1327 _mm_unpackhi_epi64(a, b).simd_into(token),
1328 )
1329 }
1330 );
1331 kernel(self, a, b)
1332 }
1333 #[inline(always)]
1334 fn select_i8x16(self, a: mask8x16<Self>, b: i8x16<Self>, c: i8x16<Self>) -> i8x16<Self> {
1335 crate::kernel!(
1336 #[inline(always)]
1337 fn kernel(
1338 token: Avx2,
1339 a: mask8x16<Avx2>,
1340 b: i8x16<Avx2>,
1341 c: i8x16<Avx2>,
1342 ) -> i8x16<Avx2> {
1343 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
1344 }
1345 );
1346 kernel(self, a, b, c)
1347 }
1348 #[inline(always)]
1349 fn combine_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x32<Self> {
1350 crate::kernel!(
1351 #[inline(always)]
1352 fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x32<Avx2> {
1353 _mm256_setr_m128i(a.into(), b.into()).simd_into(token)
1354 }
1355 );
1356 kernel(self, a, b)
1357 }
1358 #[inline(always)]
1359 fn neg_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
1360 crate::kernel!(
1361 #[inline(always)]
1362 fn kernel(token: Avx2, a: i8x16<Avx2>) -> i8x16<Avx2> {
1363 _mm_sub_epi8(_mm_setzero_si128(), a.into()).simd_into(token)
1364 }
1365 );
1366 kernel(self, a)
1367 }
1368 #[inline(always)]
1369 fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16<Self>; 4usize] {
1370 crate::kernel!(
1371 #[inline(always)]
1372 fn kernel(token: Avx2, src: &[i8; 64usize]) -> [i8x16<Avx2>; 4usize] {
1373 let (chunks, []) = src.as_chunks::<16usize>() else {
1374 unreachable!()
1375 };
1376 let v0: __m128i =
1377 crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[0]);
1378 let v1: __m128i =
1379 crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[1]);
1380 let v2: __m128i =
1381 crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[2]);
1382 let v3: __m128i =
1383 crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[3]);
1384 let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
1385 let v0 = _mm_shuffle_epi8(v0, mask);
1386 let v1 = _mm_shuffle_epi8(v1, mask);
1387 let v2 = _mm_shuffle_epi8(v2, mask);
1388 let v3 = _mm_shuffle_epi8(v3, mask);
1389 let tmp0 = _mm_unpacklo_epi32(v0, v1);
1390 let tmp1 = _mm_unpackhi_epi32(v0, v1);
1391 let tmp2 = _mm_unpacklo_epi32(v2, v3);
1392 let tmp3 = _mm_unpackhi_epi32(v2, v3);
1393 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
1394 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
1395 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
1396 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
1397 [
1398 out0.simd_into(token),
1399 out1.simd_into(token),
1400 out2.simd_into(token),
1401 out3.simd_into(token),
1402 ]
1403 }
1404 );
1405 kernel(self, src)
1406 }
1407 #[inline(always)]
1408 fn store_four_interleaved_i8x16(
1409 self,
1410 vectors: [i8x16<Self>; 4usize],
1411 dest: &mut [i8; 64usize],
1412 ) -> () {
1413 crate::kernel!(
1414 #[inline(always)]
1415 fn kernel(token: Avx2, vectors: [i8x16<Avx2>; 4usize], dest: &mut [i8; 64usize]) -> () {
1416 let _ = token;
1417 let v0: __m128i = vectors[0].into();
1418 let v1: __m128i = vectors[1].into();
1419 let v2: __m128i = vectors[2].into();
1420 let v3: __m128i = vectors[3].into();
1421 let tmp0 = _mm_unpacklo_epi32(v0, v1);
1422 let tmp1 = _mm_unpackhi_epi32(v0, v1);
1423 let tmp2 = _mm_unpacklo_epi32(v2, v3);
1424 let tmp3 = _mm_unpackhi_epi32(v2, v3);
1425 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
1426 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
1427 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
1428 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
1429 let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
1430 let out0 = _mm_shuffle_epi8(out0, mask);
1431 let out1 = _mm_shuffle_epi8(out1, mask);
1432 let out2 = _mm_shuffle_epi8(out2, mask);
1433 let out3 = _mm_shuffle_epi8(out3, mask);
1434 let (chunks, []) = dest.as_chunks_mut::<16usize>() else {
1435 unreachable!()
1436 };
1437 crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>(
1438 out0,
1439 &mut chunks[0],
1440 );
1441 crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>(
1442 out1,
1443 &mut chunks[1],
1444 );
1445 crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>(
1446 out2,
1447 &mut chunks[2],
1448 );
1449 crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>(
1450 out3,
1451 &mut chunks[3],
1452 );
1453 }
1454 );
1455 kernel(self, vectors, dest);
1456 }
1457 #[inline(always)]
1458 fn widen_i8x16(self, a: i8x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
1459 crate::kernel!(
1460 #[inline(always)]
1461 fn kernel(token: Avx2, a: i8x16<Avx2>) -> (i16x8<Avx2>, i16x8<Avx2>) {
1462 let raw = a.into();
1463 (
1464 _mm_cvtepi8_epi16(raw).simd_into(token),
1465 _mm_cvtepi8_epi16(_mm_srli_si128::<8>(raw)).simd_into(token),
1466 )
1467 }
1468 );
1469 kernel(self, a)
1470 }
1471 #[inline(always)]
1472 fn widening_mul_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
1473 let (a0, a1) = self.widen_i8x16(a);
1474 let (b0, b1) = self.widen_i8x16(b);
1475 (self.mul_i16x8(a0, b0), self.mul_i16x8(a1, b1))
1476 }
1477 #[inline(always)]
1478 fn widening_add_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
1479 crate::kernel!(
1480 #[inline(always)]
1481 fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> (i16x8<Avx2>, i16x8<Avx2>) {
1482 let a = a.into();
1483 let b = b.into();
1484 let low = a;
1485 let high = b;
1486 let result_low = _mm_unpacklo_epi8(low, high);
1487 let result_high = _mm_unpackhi_epi8(low, high);
1488 let ones = _mm_set1_epi8(1);
1489 let result_low = _mm_maddubs_epi16(ones, result_low);
1490 let result_high = _mm_maddubs_epi16(ones, result_high);
1491 (result_low.simd_into(token), result_high.simd_into(token))
1492 }
1493 );
1494 kernel(self, a, b)
1495 }
1496 #[inline(always)]
1497 fn pairwise_widening_add_i8x16(self, a: i8x16<Self>) -> i16x8<Self> {
1498 crate::kernel!(
1499 #[inline(always)]
1500 fn kernel(token: Avx2, a: i8x16<Avx2>) -> i16x8<Avx2> {
1501 ({ _mm_maddubs_epi16(_mm_set1_epi8(1), a.into()) }).simd_into(token)
1502 }
1503 );
1504 kernel(self, a)
1505 }
1506 #[inline(always)]
1507 fn pairwise_dot_product_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i16x8<Self> {
1508 let (low, high) = self.widening_mul_i8x16(a, b);
1509 let (even, odd) = self.deinterleave_i16x8(low, high);
1510 self.add_i16x8(even, odd)
1511 }
1512 #[inline(always)]
1513 fn splat_u8x16(self, val: u8) -> u8x16<Self> {
1514 crate::kernel!(
1515 #[inline(always)]
1516 fn kernel(token: Avx2, val: u8) -> u8x16<Avx2> {
1517 _mm_set1_epi8(val.cast_signed()).simd_into(token)
1518 }
1519 );
1520 kernel(self, val)
1521 }
1522 #[inline(always)]
1523 fn reverse_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
1524 let indices: u8x16<Self> =
1525 [15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0].simd_into(self);
1526 self.swizzle_dyn_u8x16(a, indices)
1527 }
1528 #[inline(always)]
1529 fn slide_u8x16<const SHIFT: usize>(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1530 if SHIFT >= 16usize {
1531 return b;
1532 }
1533 let result = dyn_alignr_128(
1534 self,
1535 Bytes::to_bytes(b).val.0,
1536 Bytes::to_bytes(a).val.0,
1537 SHIFT,
1538 );
1539 Bytes::from_bytes(u8x16 {
1540 val: crate::support::Aligned128(result),
1541 simd: self,
1542 })
1543 }
1544 #[inline(always)]
1545 fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
1546 crate::kernel!(
1547 #[inline(always)]
1548 fn kernel(token: Avx2, a: u8x16<Avx2>, indices: u8x16<Avx2>) -> u8x16<Avx2> {
1549 let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, indices.into());
1550 Bytes::from_bytes(u8x16 {
1551 val: crate::support::Aligned128(result),
1552 simd: token,
1553 })
1554 }
1555 );
1556 kernel(self, a, indices)
1557 }
1558 #[inline(always)]
1559 fn swizzle_dyn_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
1560 crate::kernel!(
1561 #[inline(always)]
1562 fn kernel(token: Avx2, a: u8x16<Avx2>, indices: u8x16<Avx2>) -> u8x16<Avx2> {
1563 let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, indices.into());
1564 Bytes::from_bytes(u8x16 {
1565 val: crate::support::Aligned128(result),
1566 simd: token,
1567 })
1568 }
1569 );
1570 kernel(self, a, indices)
1571 }
1572 #[inline(always)]
1573 fn swizzle_dyn_precise_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
1574 crate::kernel!(
1575 #[inline(always)]
1576 fn kernel(token: Avx2, a: u8x16<Avx2>, indices: u8x16<Avx2>) -> u8x16<Avx2> {
1577 let indices = indices.into();
1578 let zeroing_indices = _mm_adds_epu8(indices, _mm_set1_epi8(0x70));
1579 let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices);
1580 let result_bytes = u8x16 {
1581 val: crate::support::Aligned128(result),
1582 simd: token,
1583 };
1584 Bytes::from_bytes(result_bytes)
1585 }
1586 );
1587 kernel(self, a, indices)
1588 }
1589 #[inline(always)]
1590 fn concat_swizzle_dyn_u8x16(
1591 self,
1592 a: u8x16<Self>,
1593 b: u8x16<Self>,
1594 indices: u8x16<Self>,
1595 ) -> u8x16<Self> {
1596 self.concat_swizzle_dyn_precise_u8x16(a, b, indices)
1597 }
1598 #[inline(always)]
1599 fn concat_swizzle_dyn_precise_u8x16(
1600 self,
1601 a: u8x16<Self>,
1602 b: u8x16<Self>,
1603 indices: u8x16<Self>,
1604 ) -> u8x16<Self> {
1605 let first_table = Bytes::to_bytes(a);
1606 let second_table = Bytes::to_bytes(b);
1607 let second_table_offset = self.splat_u8x16(16);
1608 let from_first = self.swizzle_dyn_precise_u8x16(first_table, indices);
1609 let from_second = self
1610 .swizzle_dyn_precise_u8x16(second_table, self.sub_u8x16(indices, second_table_offset));
1611 let result_bytes = self.or_u8x16(from_first, from_second);
1612 Bytes::from_bytes(result_bytes)
1613 }
1614 #[inline(always)]
1615 fn compress_u8x16(self, values: u8x16<Self>, mask: mask8x16<Self>) -> u8x16<Self> {
1616 crate::kernel!(
1617 #[inline(always)]
1618 fn kernel(token: Avx2, values: u8x16<Avx2>, mask: mask8x16<Avx2>) -> u8x16<Avx2> {
1619 let mask_bits = token.to_bitmask_mask8x16(mask) as u16;
1620 let low_mask = (usize::from(mask_bits)) & 0xff;
1621 let high_mask = (usize::from(mask_bits)) >> 8;
1622 let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask];
1623 let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask];
1624 let high_control = high_control | 0x0808_0808_0808_0808;
1625 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
1626 let low_count = low_mask.count_ones() as usize;
1627 let compacted = _mm_shuffle_epi8(values.into(), control);
1628 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count];
1629 let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
1630 let compressed = _mm_shuffle_epi8(compacted, splice);
1631 compressed.simd_into(token)
1632 }
1633 );
1634 kernel(self, values, mask)
1635 }
1636 #[inline(always)]
1637 fn compress_merge_u8x16(
1638 self,
1639 values: u8x16<Self>,
1640 mask: mask8x16<Self>,
1641 merge: u8x16<Self>,
1642 ) -> u8x16<Self> {
1643 crate::kernel!(
1644 #[inline(always)]
1645 fn kernel(
1646 token: Avx2,
1647 values: u8x16<Avx2>,
1648 mask: mask8x16<Avx2>,
1649 merge: u8x16<Avx2>,
1650 ) -> u8x16<Avx2> {
1651 let mask_bits = token.to_bitmask_mask8x16(mask) as u16;
1652 let low_mask = (usize::from(mask_bits)) & 0xff;
1653 let high_mask = (usize::from(mask_bits)) >> 8;
1654 let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask];
1655 let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask];
1656 let high_control = high_control | 0x0808_0808_0808_0808;
1657 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
1658 let low_count = low_mask.count_ones() as usize;
1659 let compacted = _mm_shuffle_epi8(values.into(), control);
1660 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count];
1661 let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
1662 let compressed = _mm_shuffle_epi8(compacted, splice);
1663 let count = mask_bits.count_ones() as usize;
1664 let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>(
1665 &crate::support::COMPACT_PREFIX_MASKS[count],
1666 );
1667 _mm_blendv_epi8(merge.into(), compressed, prefix).simd_into(token)
1668 }
1669 );
1670 kernel(self, values, mask, merge)
1671 }
1672 #[inline(always)]
1673 fn expand_u8x16(self, values: u8x16<Self>, mask: mask8x16<Self>) -> u8x16<Self> {
1674 crate::kernel!(
1675 #[inline(always)]
1676 fn kernel(token: Avx2, values: u8x16<Avx2>, mask: mask8x16<Avx2>) -> u8x16<Avx2> {
1677 let mask_bits = token.to_bitmask_mask8x16(mask) as u16;
1678 let low_mask = (usize::from(mask_bits)) & 0xff;
1679 let high_mask = (usize::from(mask_bits)) >> 8;
1680 let low_count = low_mask.count_ones() as u64;
1681 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
1682 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
1683 let high_base = low_count * 0x0101_0101_0101_0101;
1684 let high_control = high_control + high_base;
1685 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
1686 let expanded = _mm_shuffle_epi8(values.into(), control);
1687 expanded.simd_into(token)
1688 }
1689 );
1690 kernel(self, values, mask)
1691 }
1692 #[inline(always)]
1693 fn expand_merge_u8x16(
1694 self,
1695 values: u8x16<Self>,
1696 mask: mask8x16<Self>,
1697 merge: u8x16<Self>,
1698 ) -> u8x16<Self> {
1699 crate::kernel!(
1700 #[inline(always)]
1701 fn kernel(
1702 token: Avx2,
1703 values: u8x16<Avx2>,
1704 mask: mask8x16<Avx2>,
1705 merge: u8x16<Avx2>,
1706 ) -> u8x16<Avx2> {
1707 let mask_bits = token.to_bitmask_mask8x16(mask) as u16;
1708 let low_mask = (usize::from(mask_bits)) & 0xff;
1709 let high_mask = (usize::from(mask_bits)) >> 8;
1710 let low_count = low_mask.count_ones() as u64;
1711 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
1712 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
1713 let high_base = low_count * 0x0101_0101_0101_0101;
1714 let high_control = high_control + high_base;
1715 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
1716 let expanded = _mm_shuffle_epi8(values.into(), control);
1717 _mm_blendv_epi8(merge.into(), expanded, mask.into()).simd_into(token)
1718 }
1719 );
1720 kernel(self, values, mask, merge)
1721 }
1722 #[inline(always)]
1723 fn count_ones_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
1724 crate::kernel!(
1725 #[inline(always)]
1726 fn kernel(token: Avx2, a: u8x16<Avx2>) -> u8x16<Avx2> {
1727 let value = a.into();
1728 let nibble_mask = _mm_set1_epi8(0x0f);
1729 let lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4);
1730 let low = _mm_and_si128(value, nibble_mask);
1731 let high = _mm_and_si128(_mm_srli_epi16::<4>(value), nibble_mask);
1732 let byte_counts = _mm_add_epi8(
1733 _mm_shuffle_epi8(lookup, low),
1734 _mm_shuffle_epi8(lookup, high),
1735 );
1736 byte_counts.simd_into(token)
1737 }
1738 );
1739 kernel(self, a)
1740 }
1741 #[inline(always)]
1742 fn count_zeros_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
1743 self.count_ones_u8x16(self.not_u8x16(a))
1744 }
1745 #[inline(always)]
1746 fn add_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1747 crate::kernel!(
1748 #[inline(always)]
1749 fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
1750 _mm_add_epi8(a.into(), b.into()).simd_into(token)
1751 }
1752 );
1753 kernel(self, a, b)
1754 }
1755 #[inline(always)]
1756 fn saturating_add_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1757 crate::kernel!(
1758 #[inline(always)]
1759 fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
1760 _mm_adds_epu8(a.into(), b.into()).simd_into(token)
1761 }
1762 );
1763 kernel(self, a, b)
1764 }
1765 #[inline(always)]
1766 fn sub_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1767 crate::kernel!(
1768 #[inline(always)]
1769 fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
1770 _mm_sub_epi8(a.into(), b.into()).simd_into(token)
1771 }
1772 );
1773 kernel(self, a, b)
1774 }
1775 #[inline(always)]
1776 fn saturating_sub_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1777 crate::kernel!(
1778 #[inline(always)]
1779 fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
1780 _mm_subs_epu8(a.into(), b.into()).simd_into(token)
1781 }
1782 );
1783 kernel(self, a, b)
1784 }
1785 #[inline(always)]
1786 fn mul_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1787 crate::kernel!(
1788 #[inline(always)]
1789 fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
1790 let a = a.into();
1791 let b = b.into();
1792 let low_mask = _mm_set1_epi16(0xFF);
1793 let dst_even = _mm_mullo_epi16(a, b);
1794 let dst_odd = _mm_maddubs_epi16(a, _mm_andnot_si128(low_mask, b));
1795 _mm_or_si128(
1796 _mm_slli_epi16(dst_odd, 8),
1797 _mm_and_si128(dst_even, low_mask),
1798 )
1799 .simd_into(token)
1800 }
1801 );
1802 kernel(self, a, b)
1803 }
1804 #[inline(always)]
1805 fn and_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1806 crate::kernel!(
1807 #[inline(always)]
1808 fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
1809 _mm_and_si128(a.into(), b.into()).simd_into(token)
1810 }
1811 );
1812 kernel(self, a, b)
1813 }
1814 #[inline(always)]
1815 fn or_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1816 crate::kernel!(
1817 #[inline(always)]
1818 fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
1819 _mm_or_si128(a.into(), b.into()).simd_into(token)
1820 }
1821 );
1822 kernel(self, a, b)
1823 }
1824 #[inline(always)]
1825 fn xor_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1826 crate::kernel!(
1827 #[inline(always)]
1828 fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
1829 _mm_xor_si128(a.into(), b.into()).simd_into(token)
1830 }
1831 );
1832 kernel(self, a, b)
1833 }
1834 #[inline(always)]
1835 fn not_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
1836 a ^ !0
1837 }
1838 #[inline(always)]
1839 fn shl_u8x16(self, a: u8x16<Self>, shift: u32) -> u8x16<Self> {
1840 crate::kernel!(
1841 #[inline(always)]
1842 fn kernel(token: Avx2, a: u8x16<Avx2>, shift: u32) -> u8x16<Avx2> {
1843 let val = a.into();
1844 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
1845 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
1846 let byte_mask = _mm_set1_epi8(mask_byte);
1847 _mm_sll_epi16(_mm_and_si128(val, byte_mask), shift_count).simd_into(token)
1848 }
1849 );
1850 kernel(self, a, shift)
1851 }
1852 #[inline(always)]
1853 fn shlv_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1854 [
1855 u8::wrapping_shl(a[0usize], b[0usize] as u32),
1856 u8::wrapping_shl(a[1usize], b[1usize] as u32),
1857 u8::wrapping_shl(a[2usize], b[2usize] as u32),
1858 u8::wrapping_shl(a[3usize], b[3usize] as u32),
1859 u8::wrapping_shl(a[4usize], b[4usize] as u32),
1860 u8::wrapping_shl(a[5usize], b[5usize] as u32),
1861 u8::wrapping_shl(a[6usize], b[6usize] as u32),
1862 u8::wrapping_shl(a[7usize], b[7usize] as u32),
1863 u8::wrapping_shl(a[8usize], b[8usize] as u32),
1864 u8::wrapping_shl(a[9usize], b[9usize] as u32),
1865 u8::wrapping_shl(a[10usize], b[10usize] as u32),
1866 u8::wrapping_shl(a[11usize], b[11usize] as u32),
1867 u8::wrapping_shl(a[12usize], b[12usize] as u32),
1868 u8::wrapping_shl(a[13usize], b[13usize] as u32),
1869 u8::wrapping_shl(a[14usize], b[14usize] as u32),
1870 u8::wrapping_shl(a[15usize], b[15usize] as u32),
1871 ]
1872 .simd_into(self)
1873 }
1874 #[inline(always)]
1875 fn shr_u8x16(self, a: u8x16<Self>, shift: u32) -> u8x16<Self> {
1876 crate::kernel!(
1877 #[inline(always)]
1878 fn kernel(token: Avx2, a: u8x16<Avx2>, shift: u32) -> u8x16<Avx2> {
1879 let val = a.into();
1880 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
1881 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
1882 let byte_mask = _mm_set1_epi8(mask_byte);
1883 let shifted = _mm_srl_epi16(val, shift_count);
1884 let result = { _mm_and_si128(shifted, byte_mask) };
1885 result.simd_into(token)
1886 }
1887 );
1888 kernel(self, a, shift)
1889 }
1890 #[inline(always)]
1891 fn shrv_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1892 [
1893 u8::wrapping_shr(a[0usize], b[0usize] as u32),
1894 u8::wrapping_shr(a[1usize], b[1usize] as u32),
1895 u8::wrapping_shr(a[2usize], b[2usize] as u32),
1896 u8::wrapping_shr(a[3usize], b[3usize] as u32),
1897 u8::wrapping_shr(a[4usize], b[4usize] as u32),
1898 u8::wrapping_shr(a[5usize], b[5usize] as u32),
1899 u8::wrapping_shr(a[6usize], b[6usize] as u32),
1900 u8::wrapping_shr(a[7usize], b[7usize] as u32),
1901 u8::wrapping_shr(a[8usize], b[8usize] as u32),
1902 u8::wrapping_shr(a[9usize], b[9usize] as u32),
1903 u8::wrapping_shr(a[10usize], b[10usize] as u32),
1904 u8::wrapping_shr(a[11usize], b[11usize] as u32),
1905 u8::wrapping_shr(a[12usize], b[12usize] as u32),
1906 u8::wrapping_shr(a[13usize], b[13usize] as u32),
1907 u8::wrapping_shr(a[14usize], b[14usize] as u32),
1908 u8::wrapping_shr(a[15usize], b[15usize] as u32),
1909 ]
1910 .simd_into(self)
1911 }
1912 #[inline(always)]
1913 fn reduce_max_u8x16(self, a: u8x16<Self>) -> u8 {
1914 crate::kernel!(
1915 #[inline(always)]
1916 fn kernel(token: Avx2, a: u8x16<Avx2>) -> u8 {
1917 let reduced: __m128i = a.into();
1918 let shifted = _mm_srli_si128::<8>(reduced);
1919 let reduced = { _mm_max_epu8(reduced, shifted) };
1920 let shifted = _mm_srli_si128::<4>(reduced);
1921 let reduced = { _mm_max_epu8(reduced, shifted) };
1922 let shifted = _mm_srli_si128::<2>(reduced);
1923 let reduced = { _mm_max_epu8(reduced, shifted) };
1924 let shifted = _mm_srli_si128::<1>(reduced);
1925 let reduced = { _mm_max_epu8(reduced, shifted) };
1926 {
1927 let lanes: [u8; 16usize] = crate::transmute::checked_transmute_copy(&reduced);
1928 lanes[0]
1929 }
1930 }
1931 );
1932 kernel(self, a)
1933 }
1934 #[inline(always)]
1935 fn reduce_min_u8x16(self, a: u8x16<Self>) -> u8 {
1936 crate::kernel!(
1937 #[inline(always)]
1938 fn kernel(token: Avx2, a: u8x16<Avx2>) -> u8 {
1939 let reduced: __m128i = a.into();
1940 let shifted = _mm_srli_si128::<8>(reduced);
1941 let reduced = { _mm_min_epu8(reduced, shifted) };
1942 let shifted = _mm_srli_si128::<4>(reduced);
1943 let reduced = { _mm_min_epu8(reduced, shifted) };
1944 let shifted = _mm_srli_si128::<2>(reduced);
1945 let reduced = { _mm_min_epu8(reduced, shifted) };
1946 let shifted = _mm_srli_si128::<1>(reduced);
1947 let reduced = { _mm_min_epu8(reduced, shifted) };
1948 {
1949 let lanes: [u8; 16usize] = crate::transmute::checked_transmute_copy(&reduced);
1950 lanes[0]
1951 }
1952 }
1953 );
1954 kernel(self, a)
1955 }
1956 #[inline(always)]
1957 fn reduce_sum_u8x16(self, a: u8x16<Self>) -> u8 {
1958 crate::kernel!(
1959 #[inline(always)]
1960 fn kernel(token: Avx2, a: u8x16<Avx2>) -> u8 {
1961 let sum: __m128i = a.into();
1962 let sum = _mm_add_epi8(sum, _mm_srli_si128::<8>(sum));
1963 let sum = _mm_add_epi8(sum, _mm_srli_si128::<4>(sum));
1964 let sum = _mm_add_epi8(sum, _mm_srli_si128::<2>(sum));
1965 let sum = _mm_add_epi8(sum, _mm_srli_si128::<1>(sum));
1966 let lanes: [u8; 16usize] = crate::transmute::checked_transmute_copy(&sum);
1967 lanes[0]
1968 }
1969 );
1970 kernel(self, a)
1971 }
1972 #[inline(always)]
1973 fn reduce_product_u8x16(self, a: u8x16<Self>) -> u8 {
1974 crate::kernel!(
1975 #[inline(always)]
1976 fn kernel(token: Avx2, a: u8x16<Avx2>) -> u8 {
1977 let value: __m128i = a.into();
1978 let product = _mm_mullo_epi16(
1979 _mm_unpacklo_epi8(value, value),
1980 _mm_unpackhi_epi8(value, value),
1981 );
1982 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b11_10_11_10>(product));
1983 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b01_01_01_01>(product));
1984 let product = _mm_mullo_epi16(product, _mm_srli_epi32::<16>(product));
1985 let lanes: [u8; 16usize] = crate::transmute::checked_transmute_copy(&product);
1986 lanes[0]
1987 }
1988 );
1989 kernel(self, a)
1990 }
1991 #[inline(always)]
1992 fn max_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1993 crate::kernel!(
1994 #[inline(always)]
1995 fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
1996 _mm_max_epu8(a.into(), b.into()).simd_into(token)
1997 }
1998 );
1999 kernel(self, a, b)
2000 }
2001 #[inline(always)]
2002 fn min_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2003 crate::kernel!(
2004 #[inline(always)]
2005 fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
2006 _mm_min_epu8(a.into(), b.into()).simd_into(token)
2007 }
2008 );
2009 kernel(self, a, b)
2010 }
2011 #[inline(always)]
2012 fn clamp_u8x16(self, a: u8x16<Self>, b: u8x16<Self>, c: u8x16<Self>) -> u8x16<Self> {
2013 debug_assert!(
2014 b.simd_le(c).all_true(),
2015 "clamp requires min <= max for all elements"
2016 );
2017 a.max(b).min(c)
2018 }
2019 #[inline(always)]
2020 fn simd_eq_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
2021 crate::kernel!(
2022 #[inline(always)]
2023 fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> mask8x16<Avx2> {
2024 _mm_cmpeq_epi8(a.into(), b.into()).simd_into(token)
2025 }
2026 );
2027 kernel(self, a, b)
2028 }
2029 #[inline(always)]
2030 fn simd_lt_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
2031 crate::kernel!(
2032 #[inline(always)]
2033 fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> mask8x16<Avx2> {
2034 {
2035 let sign_bit = _mm_set1_epi8(0x80u8.cast_signed());
2036 let lhs_signed = _mm_xor_si128(b.into(), sign_bit);
2037 let rhs_signed = _mm_xor_si128(a.into(), sign_bit);
2038 _mm_cmpgt_epi8(lhs_signed, rhs_signed)
2039 }
2040 .simd_into(token)
2041 }
2042 );
2043 kernel(self, a, b)
2044 }
2045 #[inline(always)]
2046 fn simd_le_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
2047 crate::kernel!(
2048 #[inline(always)]
2049 fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> mask8x16<Avx2> {
2050 _mm_cmpeq_epi8(_mm_min_epu8(a.into(), b.into()), a.into()).simd_into(token)
2051 }
2052 );
2053 kernel(self, a, b)
2054 }
2055 #[inline(always)]
2056 fn zip_low_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2057 crate::kernel!(
2058 #[inline(always)]
2059 fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
2060 _mm_unpacklo_epi8(a.into(), b.into()).simd_into(token)
2061 }
2062 );
2063 kernel(self, a, b)
2064 }
2065 #[inline(always)]
2066 fn zip_high_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2067 crate::kernel!(
2068 #[inline(always)]
2069 fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
2070 _mm_unpackhi_epi8(a.into(), b.into()).simd_into(token)
2071 }
2072 );
2073 kernel(self, a, b)
2074 }
2075 #[inline(always)]
2076 fn unzip_low_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2077 crate::kernel!(
2078 #[inline(always)]
2079 fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
2080 let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
2081 let t1 = _mm_shuffle_epi8(a.into(), mask);
2082 let t2 = _mm_shuffle_epi8(b.into(), mask);
2083 _mm_unpacklo_epi64(t1, t2).simd_into(token)
2084 }
2085 );
2086 kernel(self, a, b)
2087 }
2088 #[inline(always)]
2089 fn unzip_high_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2090 crate::kernel!(
2091 #[inline(always)]
2092 fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
2093 let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
2094 let t1 = _mm_shuffle_epi8(a.into(), mask);
2095 let t2 = _mm_shuffle_epi8(b.into(), mask);
2096 _mm_unpackhi_epi64(t1, t2).simd_into(token)
2097 }
2098 );
2099 kernel(self, a, b)
2100 }
2101 #[inline(always)]
2102 fn interleave_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u8x16<Self>, u8x16<Self>) {
2103 (self.zip_low_u8x16(a, b), self.zip_high_u8x16(a, b))
2104 }
2105 #[inline(always)]
2106 fn deinterleave_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u8x16<Self>, u8x16<Self>) {
2107 crate::kernel!(
2108 #[inline(always)]
2109 fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> (u8x16<Avx2>, u8x16<Avx2>) {
2110 let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
2111 let a = _mm_shuffle_epi8(a.into(), mask);
2112 let b = _mm_shuffle_epi8(b.into(), mask);
2113 (
2114 _mm_unpacklo_epi64(a, b).simd_into(token),
2115 _mm_unpackhi_epi64(a, b).simd_into(token),
2116 )
2117 }
2118 );
2119 kernel(self, a, b)
2120 }
2121 #[inline(always)]
2122 fn select_u8x16(self, a: mask8x16<Self>, b: u8x16<Self>, c: u8x16<Self>) -> u8x16<Self> {
2123 crate::kernel!(
2124 #[inline(always)]
2125 fn kernel(
2126 token: Avx2,
2127 a: mask8x16<Avx2>,
2128 b: u8x16<Avx2>,
2129 c: u8x16<Avx2>,
2130 ) -> u8x16<Avx2> {
2131 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
2132 }
2133 );
2134 kernel(self, a, b, c)
2135 }
2136 #[inline(always)]
2137 fn combine_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x32<Self> {
2138 crate::kernel!(
2139 #[inline(always)]
2140 fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x32<Avx2> {
2141 _mm256_setr_m128i(a.into(), b.into()).simd_into(token)
2142 }
2143 );
2144 kernel(self, a, b)
2145 }
2146 #[inline(always)]
2147 fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16<Self>; 4usize] {
2148 crate::kernel!(
2149 #[inline(always)]
2150 fn kernel(token: Avx2, src: &[u8; 64usize]) -> [u8x16<Avx2>; 4usize] {
2151 let (chunks, []) = src.as_chunks::<16usize>() else {
2152 unreachable!()
2153 };
2154 let v0: __m128i =
2155 crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[0]);
2156 let v1: __m128i =
2157 crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[1]);
2158 let v2: __m128i =
2159 crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[2]);
2160 let v3: __m128i =
2161 crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[3]);
2162 let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
2163 let v0 = _mm_shuffle_epi8(v0, mask);
2164 let v1 = _mm_shuffle_epi8(v1, mask);
2165 let v2 = _mm_shuffle_epi8(v2, mask);
2166 let v3 = _mm_shuffle_epi8(v3, mask);
2167 let tmp0 = _mm_unpacklo_epi32(v0, v1);
2168 let tmp1 = _mm_unpackhi_epi32(v0, v1);
2169 let tmp2 = _mm_unpacklo_epi32(v2, v3);
2170 let tmp3 = _mm_unpackhi_epi32(v2, v3);
2171 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
2172 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
2173 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
2174 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
2175 [
2176 out0.simd_into(token),
2177 out1.simd_into(token),
2178 out2.simd_into(token),
2179 out3.simd_into(token),
2180 ]
2181 }
2182 );
2183 kernel(self, src)
2184 }
2185 #[inline(always)]
2186 fn store_four_interleaved_u8x16(
2187 self,
2188 vectors: [u8x16<Self>; 4usize],
2189 dest: &mut [u8; 64usize],
2190 ) -> () {
2191 crate::kernel!(
2192 #[inline(always)]
2193 fn kernel(token: Avx2, vectors: [u8x16<Avx2>; 4usize], dest: &mut [u8; 64usize]) -> () {
2194 let _ = token;
2195 let v0: __m128i = vectors[0].into();
2196 let v1: __m128i = vectors[1].into();
2197 let v2: __m128i = vectors[2].into();
2198 let v3: __m128i = vectors[3].into();
2199 let tmp0 = _mm_unpacklo_epi32(v0, v1);
2200 let tmp1 = _mm_unpackhi_epi32(v0, v1);
2201 let tmp2 = _mm_unpacklo_epi32(v2, v3);
2202 let tmp3 = _mm_unpackhi_epi32(v2, v3);
2203 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
2204 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
2205 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
2206 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
2207 let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
2208 let out0 = _mm_shuffle_epi8(out0, mask);
2209 let out1 = _mm_shuffle_epi8(out1, mask);
2210 let out2 = _mm_shuffle_epi8(out2, mask);
2211 let out3 = _mm_shuffle_epi8(out3, mask);
2212 let (chunks, []) = dest.as_chunks_mut::<16usize>() else {
2213 unreachable!()
2214 };
2215 crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>(
2216 out0,
2217 &mut chunks[0],
2218 );
2219 crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>(
2220 out1,
2221 &mut chunks[1],
2222 );
2223 crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>(
2224 out2,
2225 &mut chunks[2],
2226 );
2227 crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>(
2228 out3,
2229 &mut chunks[3],
2230 );
2231 }
2232 );
2233 kernel(self, vectors, dest);
2234 }
2235 #[inline(always)]
2236 fn widen_u8x16(self, a: u8x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
2237 crate::kernel!(
2238 #[inline(always)]
2239 fn kernel(token: Avx2, a: u8x16<Avx2>) -> (u16x8<Avx2>, u16x8<Avx2>) {
2240 let raw = a.into();
2241 (
2242 _mm_cvtepu8_epi16(raw).simd_into(token),
2243 _mm_cvtepu8_epi16(_mm_srli_si128::<8>(raw)).simd_into(token),
2244 )
2245 }
2246 );
2247 kernel(self, a)
2248 }
2249 #[inline(always)]
2250 fn widening_mul_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
2251 let (a0, a1) = self.widen_u8x16(a);
2252 let (b0, b1) = self.widen_u8x16(b);
2253 (self.mul_u16x8(a0, b0), self.mul_u16x8(a1, b1))
2254 }
2255 #[inline(always)]
2256 fn widening_add_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
2257 crate::kernel!(
2258 #[inline(always)]
2259 fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> (u16x8<Avx2>, u16x8<Avx2>) {
2260 let a = a.into();
2261 let b = b.into();
2262 let low = a;
2263 let high = b;
2264 let result_low = _mm_unpacklo_epi8(low, high);
2265 let result_high = _mm_unpackhi_epi8(low, high);
2266 let ones = _mm_set1_epi8(1);
2267 let result_low = _mm_maddubs_epi16(result_low, ones);
2268 let result_high = _mm_maddubs_epi16(result_high, ones);
2269 (result_low.simd_into(token), result_high.simd_into(token))
2270 }
2271 );
2272 kernel(self, a, b)
2273 }
2274 #[inline(always)]
2275 fn pairwise_widening_add_u8x16(self, a: u8x16<Self>) -> u16x8<Self> {
2276 crate::kernel!(
2277 #[inline(always)]
2278 fn kernel(token: Avx2, a: u8x16<Avx2>) -> u16x8<Avx2> {
2279 ({ _mm_maddubs_epi16(a.into(), _mm_set1_epi8(1)) }).simd_into(token)
2280 }
2281 );
2282 kernel(self, a)
2283 }
2284 #[inline(always)]
2285 fn pairwise_dot_product_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u16x8<Self> {
2286 let (low, high) = self.widening_mul_u8x16(a, b);
2287 let (even, odd) = self.deinterleave_u16x8(low, high);
2288 self.add_u16x8(even, odd)
2289 }
2290 #[inline(always)]
2291 fn splat_mask8x16(self, val: bool) -> mask8x16<Self> {
2292 crate::kernel!(
2293 #[inline(always)]
2294 fn kernel(token: Avx2, val: bool) -> mask8x16<Avx2> {
2295 let val: i8 = if val { !0 } else { 0 };
2296 _mm_set1_epi8(val).simd_into(token)
2297 }
2298 );
2299 kernel(self, val)
2300 }
2301 #[inline(always)]
2302 fn from_bitmask_mask8x16(self, bits: u64) -> mask8x16<Self> {
2303 crate::kernel!(
2304 #[inline(always)]
2305 fn kernel(token: Avx2, bits: u64) -> mask8x16<Avx2> {
2306 {
2307 let bit_bytes = _mm_cvtsi32_si128(bits as i32);
2308 let bit_bytes = _mm_shuffle_epi8(
2309 bit_bytes,
2310 _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1),
2311 );
2312 let bit_mask =
2313 _mm_setr_epi8(1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128);
2314 _mm_cmpeq_epi8(_mm_and_si128(bit_bytes, bit_mask), bit_mask)
2315 }
2316 .simd_into(token)
2317 }
2318 );
2319 kernel(self, bits)
2320 }
2321 #[inline(always)]
2322 fn to_bitmask_mask8x16(self, a: mask8x16<Self>) -> u64 {
2323 crate::kernel!(
2324 #[inline(always)]
2325 fn kernel(token: Avx2, a: mask8x16<Avx2>) -> u64 {
2326 _mm_movemask_epi8(a.into()) as u32 as u64
2327 }
2328 );
2329 kernel(self, a)
2330 }
2331 #[inline(always)]
2332 fn from_vector_mask8x16(self, vector: i8x16<Self>) -> mask8x16<Self> {
2333 mask8x16 {
2334 val: vector.val,
2335 simd: self,
2336 }
2337 }
2338 #[inline(always)]
2339 fn to_vector_mask8x16(self, a: mask8x16<Self>) -> i8x16<Self> {
2340 i8x16 {
2341 val: a.val,
2342 simd: self,
2343 }
2344 }
2345 #[inline(always)]
2346 fn set_mask8x16(self, a: &mut mask8x16<Self>, index: usize, value: bool) -> () {
2347 assert!(
2348 index < 16usize,
2349 "mask lane index {index} is out of bounds for {} lanes",
2350 16usize
2351 );
2352 let mut lanes: [i8; 16usize] = (*a).into();
2353 lanes[index] = if value { !0 } else { 0 };
2354 *a = lanes.simd_into(self);
2355 }
2356 #[inline(always)]
2357 fn rotate_elements_left_mask8x16<const OFFSET: usize>(
2358 self,
2359 a: mask8x16<Self>,
2360 ) -> mask8x16<Self> {
2361 let int = i8x16 {
2362 val: crate::transmute::checked_transmute_copy(&a.val),
2363 simd: self,
2364 };
2365 let rotated = int.rotate_elements_left::<OFFSET>();
2366 mask8x16 {
2367 val: crate::transmute::checked_transmute_copy(&rotated.val),
2368 simd: self,
2369 }
2370 }
2371 #[inline(always)]
2372 fn rotate_elements_right_mask8x16<const OFFSET: usize>(
2373 self,
2374 a: mask8x16<Self>,
2375 ) -> mask8x16<Self> {
2376 let int = i8x16 {
2377 val: crate::transmute::checked_transmute_copy(&a.val),
2378 simd: self,
2379 };
2380 let rotated = int.rotate_elements_right::<OFFSET>();
2381 mask8x16 {
2382 val: crate::transmute::checked_transmute_copy(&rotated.val),
2383 simd: self,
2384 }
2385 }
2386 #[inline(always)]
2387 fn and_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
2388 crate::kernel!(
2389 #[inline(always)]
2390 fn kernel(token: Avx2, a: mask8x16<Avx2>, b: mask8x16<Avx2>) -> mask8x16<Avx2> {
2391 _mm_and_si128(a.into(), b.into()).simd_into(token)
2392 }
2393 );
2394 kernel(self, a, b)
2395 }
2396 #[inline(always)]
2397 fn or_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
2398 crate::kernel!(
2399 #[inline(always)]
2400 fn kernel(token: Avx2, a: mask8x16<Avx2>, b: mask8x16<Avx2>) -> mask8x16<Avx2> {
2401 _mm_or_si128(a.into(), b.into()).simd_into(token)
2402 }
2403 );
2404 kernel(self, a, b)
2405 }
2406 #[inline(always)]
2407 fn xor_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
2408 crate::kernel!(
2409 #[inline(always)]
2410 fn kernel(token: Avx2, a: mask8x16<Avx2>, b: mask8x16<Avx2>) -> mask8x16<Avx2> {
2411 _mm_xor_si128(a.into(), b.into()).simd_into(token)
2412 }
2413 );
2414 kernel(self, a, b)
2415 }
2416 #[inline(always)]
2417 fn not_mask8x16(self, a: mask8x16<Self>) -> mask8x16<Self> {
2418 self.xor_mask8x16(a, self.splat_mask8x16(true))
2419 }
2420 #[inline(always)]
2421 fn reverse_mask8x16(self, a: mask8x16<Self>) -> mask8x16<Self> {
2422 let lanes = i8x16 {
2423 val: crate::transmute::checked_transmute_copy(&a.val),
2424 simd: self,
2425 };
2426 let reversed = self.reverse_i8x16(lanes);
2427 mask8x16 {
2428 val: crate::transmute::checked_transmute_copy(&reversed.val),
2429 simd: self,
2430 }
2431 }
2432 #[inline(always)]
2433 fn select_mask8x16(
2434 self,
2435 a: mask8x16<Self>,
2436 b: mask8x16<Self>,
2437 c: mask8x16<Self>,
2438 ) -> mask8x16<Self> {
2439 crate::kernel!(
2440 #[inline(always)]
2441 fn kernel(
2442 token: Avx2,
2443 a: mask8x16<Avx2>,
2444 b: mask8x16<Avx2>,
2445 c: mask8x16<Avx2>,
2446 ) -> mask8x16<Avx2> {
2447 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
2448 }
2449 );
2450 kernel(self, a, b, c)
2451 }
2452 #[inline(always)]
2453 fn simd_eq_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
2454 crate::kernel!(
2455 #[inline(always)]
2456 fn kernel(token: Avx2, a: mask8x16<Avx2>, b: mask8x16<Avx2>) -> mask8x16<Avx2> {
2457 _mm_cmpeq_epi8(a.into(), b.into()).simd_into(token)
2458 }
2459 );
2460 kernel(self, a, b)
2461 }
2462 #[inline(always)]
2463 fn any_true_mask8x16(self, a: mask8x16<Self>) -> bool {
2464 crate::kernel!(
2465 #[inline(always)]
2466 fn kernel(token: Avx2, a: mask8x16<Avx2>) -> bool {
2467 _mm_movemask_epi8(a.into()) as u32 != 0
2468 }
2469 );
2470 kernel(self, a)
2471 }
2472 #[inline(always)]
2473 fn all_true_mask8x16(self, a: mask8x16<Self>) -> bool {
2474 crate::kernel!(
2475 #[inline(always)]
2476 fn kernel(token: Avx2, a: mask8x16<Avx2>) -> bool {
2477 _mm_movemask_epi8(a.into()) as u32 == 0xffff
2478 }
2479 );
2480 kernel(self, a)
2481 }
2482 #[inline(always)]
2483 fn any_false_mask8x16(self, a: mask8x16<Self>) -> bool {
2484 crate::kernel!(
2485 #[inline(always)]
2486 fn kernel(token: Avx2, a: mask8x16<Avx2>) -> bool {
2487 _mm_movemask_epi8(a.into()) as u32 != 0xffff
2488 }
2489 );
2490 kernel(self, a)
2491 }
2492 #[inline(always)]
2493 fn all_false_mask8x16(self, a: mask8x16<Self>) -> bool {
2494 crate::kernel!(
2495 #[inline(always)]
2496 fn kernel(token: Avx2, a: mask8x16<Avx2>) -> bool {
2497 _mm_movemask_epi8(a.into()) as u32 == 0
2498 }
2499 );
2500 kernel(self, a)
2501 }
2502 #[inline(always)]
2503 fn combine_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x32<Self> {
2504 crate::kernel!(
2505 #[inline(always)]
2506 fn kernel(token: Avx2, a: mask8x16<Avx2>, b: mask8x16<Avx2>) -> mask8x32<Avx2> {
2507 _mm256_setr_m128i(a.into(), b.into()).simd_into(token)
2508 }
2509 );
2510 kernel(self, a, b)
2511 }
2512 #[inline(always)]
2513 fn widen_mask8x16(self, a: mask8x16<Self>) -> (mask16x8<Self>, mask16x8<Self>) {
2514 crate::kernel!(
2515 #[inline(always)]
2516 fn kernel(token: Avx2, a: mask8x16<Avx2>) -> (mask16x8<Avx2>, mask16x8<Avx2>) {
2517 let raw = a.into();
2518 (
2519 _mm_unpacklo_epi8(raw, raw).simd_into(token),
2520 _mm_unpackhi_epi8(raw, raw).simd_into(token),
2521 )
2522 }
2523 );
2524 kernel(self, a)
2525 }
2526 #[inline(always)]
2527 fn abs_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2528 crate::kernel!(
2529 #[inline(always)]
2530 fn kernel(token: Avx2, a: i16x8<Avx2>) -> i16x8<Avx2> {
2531 _mm_abs_epi16(a.into()).simd_into(token)
2532 }
2533 );
2534 kernel(self, a)
2535 }
2536 #[inline(always)]
2537 fn splat_i16x8(self, val: i16) -> i16x8<Self> {
2538 crate::kernel!(
2539 #[inline(always)]
2540 fn kernel(token: Avx2, val: i16) -> i16x8<Avx2> {
2541 _mm_set1_epi16(val).simd_into(token)
2542 }
2543 );
2544 kernel(self, val)
2545 }
2546 #[inline(always)]
2547 fn reverse_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2548 let indices: u8x16<Self> =
2549 [14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1].simd_into(self);
2550 self.swizzle_dyn_i16x8(a, indices)
2551 }
2552 #[inline(always)]
2553 fn slide_i16x8<const SHIFT: usize>(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2554 if SHIFT >= 8usize {
2555 return b;
2556 }
2557 let result = dyn_alignr_128(
2558 self,
2559 Bytes::to_bytes(b).val.0,
2560 Bytes::to_bytes(a).val.0,
2561 SHIFT * 2usize,
2562 );
2563 Bytes::from_bytes(u8x16 {
2564 val: crate::support::Aligned128(result),
2565 simd: self,
2566 })
2567 }
2568 #[inline(always)]
2569 fn compress_i16x8(self, values: i16x8<Self>, mask: mask16x8<Self>) -> i16x8<Self> {
2570 let bits = self.to_bitmask_mask16x8(mask) as usize & 255;
2571 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0);
2572 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
2573 }
2574 #[inline(always)]
2575 fn compress_merge_i16x8(
2576 self,
2577 values: i16x8<Self>,
2578 mask: mask16x8<Self>,
2579 merge: i16x8<Self>,
2580 ) -> i16x8<Self> {
2581 let bits = self.to_bitmask_mask16x8(mask) as usize & 255;
2582 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0);
2583 let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control);
2584 let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0));
2585 Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed))
2586 }
2587 #[inline(always)]
2588 fn expand_i16x8(self, values: i16x8<Self>, mask: mask16x8<Self>) -> i16x8<Self> {
2589 let bits = self.to_bitmask_mask16x8(mask) as usize & 255;
2590 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0);
2591 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
2592 }
2593 #[inline(always)]
2594 fn expand_merge_i16x8(
2595 self,
2596 values: i16x8<Self>,
2597 mask: mask16x8<Self>,
2598 merge: i16x8<Self>,
2599 ) -> i16x8<Self> {
2600 let bits = self.to_bitmask_mask16x8(mask) as usize & 255;
2601 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0);
2602 let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control));
2603 self.select_i16x8(mask, expanded, merge)
2604 }
2605 #[inline(always)]
2606 fn count_ones_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2607 crate::kernel!(
2608 #[inline(always)]
2609 fn kernel(token: Avx2, a: i16x8<Avx2>) -> i16x8<Avx2> {
2610 let value = a.into();
2611 let nibble_mask = _mm_set1_epi8(0x0f);
2612 let lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4);
2613 let low = _mm_and_si128(value, nibble_mask);
2614 let high = _mm_and_si128(_mm_srli_epi16::<4>(value), nibble_mask);
2615 let byte_counts = _mm_add_epi8(
2616 _mm_shuffle_epi8(lookup, low),
2617 _mm_shuffle_epi8(lookup, high),
2618 );
2619 _mm_maddubs_epi16(byte_counts, _mm_set1_epi8(1)).simd_into(token)
2620 }
2621 );
2622 kernel(self, a)
2623 }
2624 #[inline(always)]
2625 fn count_zeros_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2626 self.count_ones_i16x8(self.not_i16x8(a))
2627 }
2628 #[inline(always)]
2629 fn add_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2630 crate::kernel!(
2631 #[inline(always)]
2632 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
2633 _mm_add_epi16(a.into(), b.into()).simd_into(token)
2634 }
2635 );
2636 kernel(self, a, b)
2637 }
2638 #[inline(always)]
2639 fn saturating_add_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2640 crate::kernel!(
2641 #[inline(always)]
2642 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
2643 _mm_adds_epi16(a.into(), b.into()).simd_into(token)
2644 }
2645 );
2646 kernel(self, a, b)
2647 }
2648 #[inline(always)]
2649 fn sub_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2650 crate::kernel!(
2651 #[inline(always)]
2652 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
2653 _mm_sub_epi16(a.into(), b.into()).simd_into(token)
2654 }
2655 );
2656 kernel(self, a, b)
2657 }
2658 #[inline(always)]
2659 fn saturating_sub_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2660 crate::kernel!(
2661 #[inline(always)]
2662 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
2663 _mm_subs_epi16(a.into(), b.into()).simd_into(token)
2664 }
2665 );
2666 kernel(self, a, b)
2667 }
2668 #[inline(always)]
2669 fn mul_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2670 crate::kernel!(
2671 #[inline(always)]
2672 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
2673 _mm_mullo_epi16(a.into(), b.into()).simd_into(token)
2674 }
2675 );
2676 kernel(self, a, b)
2677 }
2678 #[inline(always)]
2679 fn and_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2680 crate::kernel!(
2681 #[inline(always)]
2682 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
2683 _mm_and_si128(a.into(), b.into()).simd_into(token)
2684 }
2685 );
2686 kernel(self, a, b)
2687 }
2688 #[inline(always)]
2689 fn or_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2690 crate::kernel!(
2691 #[inline(always)]
2692 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
2693 _mm_or_si128(a.into(), b.into()).simd_into(token)
2694 }
2695 );
2696 kernel(self, a, b)
2697 }
2698 #[inline(always)]
2699 fn xor_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2700 crate::kernel!(
2701 #[inline(always)]
2702 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
2703 _mm_xor_si128(a.into(), b.into()).simd_into(token)
2704 }
2705 );
2706 kernel(self, a, b)
2707 }
2708 #[inline(always)]
2709 fn not_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2710 a ^ !0
2711 }
2712 #[inline(always)]
2713 fn shl_i16x8(self, a: i16x8<Self>, shift: u32) -> i16x8<Self> {
2714 crate::kernel!(
2715 #[inline(always)]
2716 fn kernel(token: Avx2, a: i16x8<Avx2>, shift: u32) -> i16x8<Avx2> {
2717 _mm_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
2718 }
2719 );
2720 kernel(self, a, shift)
2721 }
2722 #[inline(always)]
2723 fn shlv_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2724 [
2725 i16::wrapping_shl(a[0usize], b[0usize] as u32),
2726 i16::wrapping_shl(a[1usize], b[1usize] as u32),
2727 i16::wrapping_shl(a[2usize], b[2usize] as u32),
2728 i16::wrapping_shl(a[3usize], b[3usize] as u32),
2729 i16::wrapping_shl(a[4usize], b[4usize] as u32),
2730 i16::wrapping_shl(a[5usize], b[5usize] as u32),
2731 i16::wrapping_shl(a[6usize], b[6usize] as u32),
2732 i16::wrapping_shl(a[7usize], b[7usize] as u32),
2733 ]
2734 .simd_into(self)
2735 }
2736 #[inline(always)]
2737 fn shr_i16x8(self, a: i16x8<Self>, shift: u32) -> i16x8<Self> {
2738 crate::kernel!(
2739 #[inline(always)]
2740 fn kernel(token: Avx2, a: i16x8<Avx2>, shift: u32) -> i16x8<Avx2> {
2741 _mm_sra_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
2742 }
2743 );
2744 kernel(self, a, shift)
2745 }
2746 #[inline(always)]
2747 fn shrv_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2748 [
2749 i16::wrapping_shr(a[0usize], b[0usize] as u32),
2750 i16::wrapping_shr(a[1usize], b[1usize] as u32),
2751 i16::wrapping_shr(a[2usize], b[2usize] as u32),
2752 i16::wrapping_shr(a[3usize], b[3usize] as u32),
2753 i16::wrapping_shr(a[4usize], b[4usize] as u32),
2754 i16::wrapping_shr(a[5usize], b[5usize] as u32),
2755 i16::wrapping_shr(a[6usize], b[6usize] as u32),
2756 i16::wrapping_shr(a[7usize], b[7usize] as u32),
2757 ]
2758 .simd_into(self)
2759 }
2760 #[inline(always)]
2761 fn reduce_max_i16x8(self, a: i16x8<Self>) -> i16 {
2762 crate::kernel!(
2763 #[inline(always)]
2764 fn kernel(token: Avx2, a: i16x8<Avx2>) -> i16 {
2765 let reduced: __m128i = a.into();
2766 let shifted = _mm_srli_si128::<8>(reduced);
2767 let reduced = { _mm_max_epi16(reduced, shifted) };
2768 let shifted = _mm_srli_si128::<4>(reduced);
2769 let reduced = { _mm_max_epi16(reduced, shifted) };
2770 let shifted = _mm_srli_si128::<2>(reduced);
2771 let reduced = { _mm_max_epi16(reduced, shifted) };
2772 {
2773 let lanes: [i16; 8usize] = crate::transmute::checked_transmute_copy(&reduced);
2774 lanes[0]
2775 }
2776 }
2777 );
2778 kernel(self, a)
2779 }
2780 #[inline(always)]
2781 fn reduce_min_i16x8(self, a: i16x8<Self>) -> i16 {
2782 crate::kernel!(
2783 #[inline(always)]
2784 fn kernel(token: Avx2, a: i16x8<Avx2>) -> i16 {
2785 let reduced: __m128i = a.into();
2786 let shifted = _mm_srli_si128::<8>(reduced);
2787 let reduced = { _mm_min_epi16(reduced, shifted) };
2788 let shifted = _mm_srli_si128::<4>(reduced);
2789 let reduced = { _mm_min_epi16(reduced, shifted) };
2790 let shifted = _mm_srli_si128::<2>(reduced);
2791 let reduced = { _mm_min_epi16(reduced, shifted) };
2792 {
2793 let lanes: [i16; 8usize] = crate::transmute::checked_transmute_copy(&reduced);
2794 lanes[0]
2795 }
2796 }
2797 );
2798 kernel(self, a)
2799 }
2800 #[inline(always)]
2801 fn reduce_sum_i16x8(self, a: i16x8<Self>) -> i16 {
2802 crate::kernel!(
2803 #[inline(always)]
2804 fn kernel(token: Avx2, a: i16x8<Avx2>) -> i16 {
2805 let sum: __m128i = a.into();
2806 let sum = _mm_add_epi16(sum, _mm_srli_si128::<8>(sum));
2807 let sum = _mm_add_epi16(sum, _mm_srli_si128::<4>(sum));
2808 let sum = _mm_add_epi16(sum, _mm_srli_si128::<2>(sum));
2809 let lanes: [i16; 8usize] = crate::transmute::checked_transmute_copy(&sum);
2810 lanes[0]
2811 }
2812 );
2813 kernel(self, a)
2814 }
2815 #[inline(always)]
2816 fn reduce_product_i16x8(self, a: i16x8<Self>) -> i16 {
2817 crate::kernel!(
2818 #[inline(always)]
2819 fn kernel(token: Avx2, a: i16x8<Avx2>) -> i16 {
2820 let product: __m128i = a.into();
2821 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b11_10_11_10>(product));
2822 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b01_01_01_01>(product));
2823 let product = _mm_mullo_epi16(product, _mm_srli_epi32::<16>(product));
2824 let lanes: [i16; 8usize] = crate::transmute::checked_transmute_copy(&product);
2825 lanes[0]
2826 }
2827 );
2828 kernel(self, a)
2829 }
2830 #[inline(always)]
2831 fn max_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2832 crate::kernel!(
2833 #[inline(always)]
2834 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
2835 _mm_max_epi16(a.into(), b.into()).simd_into(token)
2836 }
2837 );
2838 kernel(self, a, b)
2839 }
2840 #[inline(always)]
2841 fn min_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2842 crate::kernel!(
2843 #[inline(always)]
2844 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
2845 _mm_min_epi16(a.into(), b.into()).simd_into(token)
2846 }
2847 );
2848 kernel(self, a, b)
2849 }
2850 #[inline(always)]
2851 fn clamp_i16x8(self, a: i16x8<Self>, b: i16x8<Self>, c: i16x8<Self>) -> i16x8<Self> {
2852 debug_assert!(
2853 b.simd_le(c).all_true(),
2854 "clamp requires min <= max for all elements"
2855 );
2856 a.max(b).min(c)
2857 }
2858 #[inline(always)]
2859 fn simd_eq_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
2860 crate::kernel!(
2861 #[inline(always)]
2862 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> mask16x8<Avx2> {
2863 _mm_cmpeq_epi16(a.into(), b.into()).simd_into(token)
2864 }
2865 );
2866 kernel(self, a, b)
2867 }
2868 #[inline(always)]
2869 fn simd_lt_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
2870 crate::kernel!(
2871 #[inline(always)]
2872 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> mask16x8<Avx2> {
2873 _mm_cmpgt_epi16(b.into(), a.into()).simd_into(token)
2874 }
2875 );
2876 kernel(self, a, b)
2877 }
2878 #[inline(always)]
2879 fn simd_le_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
2880 crate::kernel!(
2881 #[inline(always)]
2882 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> mask16x8<Avx2> {
2883 _mm_cmpeq_epi16(_mm_min_epi16(a.into(), b.into()), a.into()).simd_into(token)
2884 }
2885 );
2886 kernel(self, a, b)
2887 }
2888 #[inline(always)]
2889 fn zip_low_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2890 crate::kernel!(
2891 #[inline(always)]
2892 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
2893 _mm_unpacklo_epi16(a.into(), b.into()).simd_into(token)
2894 }
2895 );
2896 kernel(self, a, b)
2897 }
2898 #[inline(always)]
2899 fn zip_high_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2900 crate::kernel!(
2901 #[inline(always)]
2902 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
2903 _mm_unpackhi_epi16(a.into(), b.into()).simd_into(token)
2904 }
2905 );
2906 kernel(self, a, b)
2907 }
2908 #[inline(always)]
2909 fn unzip_low_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2910 crate::kernel!(
2911 #[inline(always)]
2912 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
2913 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
2914 let t1 = _mm_shuffle_epi8(a.into(), mask);
2915 let t2 = _mm_shuffle_epi8(b.into(), mask);
2916 _mm_unpacklo_epi64(t1, t2).simd_into(token)
2917 }
2918 );
2919 kernel(self, a, b)
2920 }
2921 #[inline(always)]
2922 fn unzip_high_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2923 crate::kernel!(
2924 #[inline(always)]
2925 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
2926 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
2927 let t1 = _mm_shuffle_epi8(a.into(), mask);
2928 let t2 = _mm_shuffle_epi8(b.into(), mask);
2929 _mm_unpackhi_epi64(t1, t2).simd_into(token)
2930 }
2931 );
2932 kernel(self, a, b)
2933 }
2934 #[inline(always)]
2935 fn interleave_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i16x8<Self>, i16x8<Self>) {
2936 (self.zip_low_i16x8(a, b), self.zip_high_i16x8(a, b))
2937 }
2938 #[inline(always)]
2939 fn deinterleave_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i16x8<Self>, i16x8<Self>) {
2940 crate::kernel!(
2941 #[inline(always)]
2942 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> (i16x8<Avx2>, i16x8<Avx2>) {
2943 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
2944 let a = _mm_shuffle_epi8(a.into(), mask);
2945 let b = _mm_shuffle_epi8(b.into(), mask);
2946 (
2947 _mm_unpacklo_epi64(a, b).simd_into(token),
2948 _mm_unpackhi_epi64(a, b).simd_into(token),
2949 )
2950 }
2951 );
2952 kernel(self, a, b)
2953 }
2954 #[inline(always)]
2955 fn select_i16x8(self, a: mask16x8<Self>, b: i16x8<Self>, c: i16x8<Self>) -> i16x8<Self> {
2956 crate::kernel!(
2957 #[inline(always)]
2958 fn kernel(
2959 token: Avx2,
2960 a: mask16x8<Avx2>,
2961 b: i16x8<Avx2>,
2962 c: i16x8<Avx2>,
2963 ) -> i16x8<Avx2> {
2964 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
2965 }
2966 );
2967 kernel(self, a, b, c)
2968 }
2969 #[inline(always)]
2970 fn combine_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x16<Self> {
2971 crate::kernel!(
2972 #[inline(always)]
2973 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x16<Avx2> {
2974 _mm256_setr_m128i(a.into(), b.into()).simd_into(token)
2975 }
2976 );
2977 kernel(self, a, b)
2978 }
2979 #[inline(always)]
2980 fn neg_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2981 crate::kernel!(
2982 #[inline(always)]
2983 fn kernel(token: Avx2, a: i16x8<Avx2>) -> i16x8<Avx2> {
2984 _mm_sub_epi16(_mm_setzero_si128(), a.into()).simd_into(token)
2985 }
2986 );
2987 kernel(self, a)
2988 }
2989 #[inline(always)]
2990 fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8<Self>; 4usize] {
2991 crate::kernel!(
2992 #[inline(always)]
2993 fn kernel(token: Avx2, src: &[i16; 32usize]) -> [i16x8<Avx2>; 4usize] {
2994 let (chunks, []) = src.as_chunks::<8usize>() else {
2995 unreachable!()
2996 };
2997 let v0: __m128i =
2998 crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[0]);
2999 let v1: __m128i =
3000 crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[1]);
3001 let v2: __m128i =
3002 crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[2]);
3003 let v3: __m128i =
3004 crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[3]);
3005 let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15);
3006 let v0 = _mm_shuffle_epi8(v0, mask);
3007 let v1 = _mm_shuffle_epi8(v1, mask);
3008 let v2 = _mm_shuffle_epi8(v2, mask);
3009 let v3 = _mm_shuffle_epi8(v3, mask);
3010 let tmp0 = _mm_unpacklo_epi32(v0, v1);
3011 let tmp1 = _mm_unpackhi_epi32(v0, v1);
3012 let tmp2 = _mm_unpacklo_epi32(v2, v3);
3013 let tmp3 = _mm_unpackhi_epi32(v2, v3);
3014 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
3015 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
3016 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
3017 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
3018 [
3019 out0.simd_into(token),
3020 out1.simd_into(token),
3021 out2.simd_into(token),
3022 out3.simd_into(token),
3023 ]
3024 }
3025 );
3026 kernel(self, src)
3027 }
3028 #[inline(always)]
3029 fn store_four_interleaved_i16x8(
3030 self,
3031 vectors: [i16x8<Self>; 4usize],
3032 dest: &mut [i16; 32usize],
3033 ) -> () {
3034 crate::kernel!(
3035 #[inline(always)]
3036 fn kernel(
3037 token: Avx2,
3038 vectors: [i16x8<Avx2>; 4usize],
3039 dest: &mut [i16; 32usize],
3040 ) -> () {
3041 let _ = token;
3042 let v0: __m128i = vectors[0].into();
3043 let v1: __m128i = vectors[1].into();
3044 let v2: __m128i = vectors[2].into();
3045 let v3: __m128i = vectors[3].into();
3046 let tmp0 = _mm_unpacklo_epi32(v0, v1);
3047 let tmp1 = _mm_unpackhi_epi32(v0, v1);
3048 let tmp2 = _mm_unpacklo_epi32(v2, v3);
3049 let tmp3 = _mm_unpackhi_epi32(v2, v3);
3050 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
3051 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
3052 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
3053 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
3054 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
3055 let out0 = _mm_shuffle_epi8(out0, mask);
3056 let out1 = _mm_shuffle_epi8(out1, mask);
3057 let out2 = _mm_shuffle_epi8(out2, mask);
3058 let out3 = _mm_shuffle_epi8(out3, mask);
3059 let (chunks, []) = dest.as_chunks_mut::<8usize>() else {
3060 unreachable!()
3061 };
3062 crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>(
3063 out0,
3064 &mut chunks[0],
3065 );
3066 crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>(
3067 out1,
3068 &mut chunks[1],
3069 );
3070 crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>(
3071 out2,
3072 &mut chunks[2],
3073 );
3074 crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>(
3075 out3,
3076 &mut chunks[3],
3077 );
3078 }
3079 );
3080 kernel(self, vectors, dest);
3081 }
3082 #[inline(always)]
3083 fn widen_i16x8(self, a: i16x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
3084 crate::kernel!(
3085 #[inline(always)]
3086 fn kernel(token: Avx2, a: i16x8<Avx2>) -> (i32x4<Avx2>, i32x4<Avx2>) {
3087 let raw = a.into();
3088 (
3089 _mm_cvtepi16_epi32(raw).simd_into(token),
3090 _mm_cvtepi16_epi32(_mm_srli_si128::<8>(raw)).simd_into(token),
3091 )
3092 }
3093 );
3094 kernel(self, a)
3095 }
3096 #[inline(always)]
3097 fn widening_mul_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
3098 crate::kernel!(
3099 #[inline(always)]
3100 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> (i32x4<Avx2>, i32x4<Avx2>) {
3101 let a = a.into();
3102 let b = b.into();
3103 let low = _mm_mullo_epi16(a, b);
3104 let high = _mm_mulhi_epi16(a, b);
3105 let result_low = _mm_unpacklo_epi16(low, high);
3106 let result_high = _mm_unpackhi_epi16(low, high);
3107 (result_low.simd_into(token), result_high.simd_into(token))
3108 }
3109 );
3110 kernel(self, a, b)
3111 }
3112 #[inline(always)]
3113 fn widening_add_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
3114 crate::kernel!(
3115 #[inline(always)]
3116 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> (i32x4<Avx2>, i32x4<Avx2>) {
3117 let a = a.into();
3118 let b = b.into();
3119 let low = a;
3120 let high = b;
3121 let result_low = _mm_unpacklo_epi16(low, high);
3122 let result_high = _mm_unpackhi_epi16(low, high);
3123 let ones = _mm_set1_epi16(1);
3124 let result_low = _mm_madd_epi16(result_low, ones);
3125 let result_high = _mm_madd_epi16(result_high, ones);
3126 (result_low.simd_into(token), result_high.simd_into(token))
3127 }
3128 );
3129 kernel(self, a, b)
3130 }
3131 #[inline(always)]
3132 fn pairwise_widening_add_i16x8(self, a: i16x8<Self>) -> i32x4<Self> {
3133 crate::kernel!(
3134 #[inline(always)]
3135 fn kernel(token: Avx2, a: i16x8<Avx2>) -> i32x4<Avx2> {
3136 ({ _mm_madd_epi16(a.into(), _mm_set1_epi16(1)) }).simd_into(token)
3137 }
3138 );
3139 kernel(self, a)
3140 }
3141 #[inline(always)]
3142 fn pairwise_dot_product_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i32x4<Self> {
3143 crate::kernel!(
3144 #[inline(always)]
3145 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i32x4<Avx2> {
3146 _mm_madd_epi16(a.into(), b.into()).simd_into(token)
3147 }
3148 );
3149 kernel(self, a, b)
3150 }
3151 #[inline(always)]
3152 fn narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
3153 crate::kernel!(
3154 #[inline(always)]
3155 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i8x16<Avx2> {
3156 let mask = _mm_set1_epi16(0xff);
3157 _mm_packus_epi16(_mm_and_si128(a.into(), mask), _mm_and_si128(b.into(), mask))
3158 .simd_into(token)
3159 }
3160 );
3161 kernel(self, a, b)
3162 }
3163 #[inline(always)]
3164 fn saturating_narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
3165 crate::kernel!(
3166 #[inline(always)]
3167 fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i8x16<Avx2> {
3168 _mm_packs_epi16(a.into(), b.into()).simd_into(token)
3169 }
3170 );
3171 kernel(self, a, b)
3172 }
3173 #[inline(always)]
3174 fn relaxed_narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
3175 debug_assert!(
3176 a.as_slice()
3177 .iter()
3178 .chain(b.as_slice())
3179 .all(|&value| { value >= i8::MIN as i16 && value <= i8::MAX as i16 }),
3180 "relaxed_narrow inputs must fit in the destination type",
3181 );
3182 self.saturating_narrow_i16x8(a, b)
3183 }
3184 #[inline(always)]
3185 fn splat_u16x8(self, val: u16) -> u16x8<Self> {
3186 crate::kernel!(
3187 #[inline(always)]
3188 fn kernel(token: Avx2, val: u16) -> u16x8<Avx2> {
3189 _mm_set1_epi16(val.cast_signed()).simd_into(token)
3190 }
3191 );
3192 kernel(self, val)
3193 }
3194 #[inline(always)]
3195 fn reverse_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
3196 let indices: u8x16<Self> =
3197 [14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1].simd_into(self);
3198 self.swizzle_dyn_u16x8(a, indices)
3199 }
3200 #[inline(always)]
3201 fn slide_u16x8<const SHIFT: usize>(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3202 if SHIFT >= 8usize {
3203 return b;
3204 }
3205 let result = dyn_alignr_128(
3206 self,
3207 Bytes::to_bytes(b).val.0,
3208 Bytes::to_bytes(a).val.0,
3209 SHIFT * 2usize,
3210 );
3211 Bytes::from_bytes(u8x16 {
3212 val: crate::support::Aligned128(result),
3213 simd: self,
3214 })
3215 }
3216 #[inline(always)]
3217 fn compress_u16x8(self, values: u16x8<Self>, mask: mask16x8<Self>) -> u16x8<Self> {
3218 let bits = self.to_bitmask_mask16x8(mask) as usize & 255;
3219 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0);
3220 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
3221 }
3222 #[inline(always)]
3223 fn compress_merge_u16x8(
3224 self,
3225 values: u16x8<Self>,
3226 mask: mask16x8<Self>,
3227 merge: u16x8<Self>,
3228 ) -> u16x8<Self> {
3229 let bits = self.to_bitmask_mask16x8(mask) as usize & 255;
3230 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0);
3231 let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control);
3232 let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0));
3233 Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed))
3234 }
3235 #[inline(always)]
3236 fn expand_u16x8(self, values: u16x8<Self>, mask: mask16x8<Self>) -> u16x8<Self> {
3237 let bits = self.to_bitmask_mask16x8(mask) as usize & 255;
3238 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0);
3239 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
3240 }
3241 #[inline(always)]
3242 fn expand_merge_u16x8(
3243 self,
3244 values: u16x8<Self>,
3245 mask: mask16x8<Self>,
3246 merge: u16x8<Self>,
3247 ) -> u16x8<Self> {
3248 let bits = self.to_bitmask_mask16x8(mask) as usize & 255;
3249 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0);
3250 let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control));
3251 self.select_u16x8(mask, expanded, merge)
3252 }
3253 #[inline(always)]
3254 fn count_ones_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
3255 crate::kernel!(
3256 #[inline(always)]
3257 fn kernel(token: Avx2, a: u16x8<Avx2>) -> u16x8<Avx2> {
3258 let value = a.into();
3259 let nibble_mask = _mm_set1_epi8(0x0f);
3260 let lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4);
3261 let low = _mm_and_si128(value, nibble_mask);
3262 let high = _mm_and_si128(_mm_srli_epi16::<4>(value), nibble_mask);
3263 let byte_counts = _mm_add_epi8(
3264 _mm_shuffle_epi8(lookup, low),
3265 _mm_shuffle_epi8(lookup, high),
3266 );
3267 _mm_maddubs_epi16(byte_counts, _mm_set1_epi8(1)).simd_into(token)
3268 }
3269 );
3270 kernel(self, a)
3271 }
3272 #[inline(always)]
3273 fn count_zeros_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
3274 self.count_ones_u16x8(self.not_u16x8(a))
3275 }
3276 #[inline(always)]
3277 fn add_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3278 crate::kernel!(
3279 #[inline(always)]
3280 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
3281 _mm_add_epi16(a.into(), b.into()).simd_into(token)
3282 }
3283 );
3284 kernel(self, a, b)
3285 }
3286 #[inline(always)]
3287 fn saturating_add_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3288 crate::kernel!(
3289 #[inline(always)]
3290 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
3291 _mm_adds_epu16(a.into(), b.into()).simd_into(token)
3292 }
3293 );
3294 kernel(self, a, b)
3295 }
3296 #[inline(always)]
3297 fn sub_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3298 crate::kernel!(
3299 #[inline(always)]
3300 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
3301 _mm_sub_epi16(a.into(), b.into()).simd_into(token)
3302 }
3303 );
3304 kernel(self, a, b)
3305 }
3306 #[inline(always)]
3307 fn saturating_sub_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3308 crate::kernel!(
3309 #[inline(always)]
3310 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
3311 _mm_subs_epu16(a.into(), b.into()).simd_into(token)
3312 }
3313 );
3314 kernel(self, a, b)
3315 }
3316 #[inline(always)]
3317 fn mul_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3318 crate::kernel!(
3319 #[inline(always)]
3320 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
3321 _mm_mullo_epi16(a.into(), b.into()).simd_into(token)
3322 }
3323 );
3324 kernel(self, a, b)
3325 }
3326 #[inline(always)]
3327 fn and_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3328 crate::kernel!(
3329 #[inline(always)]
3330 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
3331 _mm_and_si128(a.into(), b.into()).simd_into(token)
3332 }
3333 );
3334 kernel(self, a, b)
3335 }
3336 #[inline(always)]
3337 fn or_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3338 crate::kernel!(
3339 #[inline(always)]
3340 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
3341 _mm_or_si128(a.into(), b.into()).simd_into(token)
3342 }
3343 );
3344 kernel(self, a, b)
3345 }
3346 #[inline(always)]
3347 fn xor_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3348 crate::kernel!(
3349 #[inline(always)]
3350 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
3351 _mm_xor_si128(a.into(), b.into()).simd_into(token)
3352 }
3353 );
3354 kernel(self, a, b)
3355 }
3356 #[inline(always)]
3357 fn not_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
3358 a ^ !0
3359 }
3360 #[inline(always)]
3361 fn shl_u16x8(self, a: u16x8<Self>, shift: u32) -> u16x8<Self> {
3362 crate::kernel!(
3363 #[inline(always)]
3364 fn kernel(token: Avx2, a: u16x8<Avx2>, shift: u32) -> u16x8<Avx2> {
3365 _mm_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
3366 }
3367 );
3368 kernel(self, a, shift)
3369 }
3370 #[inline(always)]
3371 fn shlv_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3372 [
3373 u16::wrapping_shl(a[0usize], b[0usize] as u32),
3374 u16::wrapping_shl(a[1usize], b[1usize] as u32),
3375 u16::wrapping_shl(a[2usize], b[2usize] as u32),
3376 u16::wrapping_shl(a[3usize], b[3usize] as u32),
3377 u16::wrapping_shl(a[4usize], b[4usize] as u32),
3378 u16::wrapping_shl(a[5usize], b[5usize] as u32),
3379 u16::wrapping_shl(a[6usize], b[6usize] as u32),
3380 u16::wrapping_shl(a[7usize], b[7usize] as u32),
3381 ]
3382 .simd_into(self)
3383 }
3384 #[inline(always)]
3385 fn shr_u16x8(self, a: u16x8<Self>, shift: u32) -> u16x8<Self> {
3386 crate::kernel!(
3387 #[inline(always)]
3388 fn kernel(token: Avx2, a: u16x8<Avx2>, shift: u32) -> u16x8<Avx2> {
3389 _mm_srl_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
3390 }
3391 );
3392 kernel(self, a, shift)
3393 }
3394 #[inline(always)]
3395 fn shrv_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3396 [
3397 u16::wrapping_shr(a[0usize], b[0usize] as u32),
3398 u16::wrapping_shr(a[1usize], b[1usize] as u32),
3399 u16::wrapping_shr(a[2usize], b[2usize] as u32),
3400 u16::wrapping_shr(a[3usize], b[3usize] as u32),
3401 u16::wrapping_shr(a[4usize], b[4usize] as u32),
3402 u16::wrapping_shr(a[5usize], b[5usize] as u32),
3403 u16::wrapping_shr(a[6usize], b[6usize] as u32),
3404 u16::wrapping_shr(a[7usize], b[7usize] as u32),
3405 ]
3406 .simd_into(self)
3407 }
3408 #[inline(always)]
3409 fn reduce_max_u16x8(self, a: u16x8<Self>) -> u16 {
3410 crate::kernel!(
3411 #[inline(always)]
3412 fn kernel(token: Avx2, a: u16x8<Avx2>) -> u16 {
3413 let reduced: __m128i = a.into();
3414 let shifted = _mm_srli_si128::<8>(reduced);
3415 let reduced = { _mm_max_epu16(reduced, shifted) };
3416 let shifted = _mm_srli_si128::<4>(reduced);
3417 let reduced = { _mm_max_epu16(reduced, shifted) };
3418 let shifted = _mm_srli_si128::<2>(reduced);
3419 let reduced = { _mm_max_epu16(reduced, shifted) };
3420 {
3421 let lanes: [u16; 8usize] = crate::transmute::checked_transmute_copy(&reduced);
3422 lanes[0]
3423 }
3424 }
3425 );
3426 kernel(self, a)
3427 }
3428 #[inline(always)]
3429 fn reduce_min_u16x8(self, a: u16x8<Self>) -> u16 {
3430 crate::kernel!(
3431 #[inline(always)]
3432 fn kernel(token: Avx2, a: u16x8<Avx2>) -> u16 {
3433 let reduced: __m128i = a.into();
3434 let shifted = _mm_srli_si128::<8>(reduced);
3435 let reduced = { _mm_min_epu16(reduced, shifted) };
3436 let shifted = _mm_srli_si128::<4>(reduced);
3437 let reduced = { _mm_min_epu16(reduced, shifted) };
3438 let shifted = _mm_srli_si128::<2>(reduced);
3439 let reduced = { _mm_min_epu16(reduced, shifted) };
3440 {
3441 let lanes: [u16; 8usize] = crate::transmute::checked_transmute_copy(&reduced);
3442 lanes[0]
3443 }
3444 }
3445 );
3446 kernel(self, a)
3447 }
3448 #[inline(always)]
3449 fn reduce_sum_u16x8(self, a: u16x8<Self>) -> u16 {
3450 crate::kernel!(
3451 #[inline(always)]
3452 fn kernel(token: Avx2, a: u16x8<Avx2>) -> u16 {
3453 let sum: __m128i = a.into();
3454 let sum = _mm_add_epi16(sum, _mm_srli_si128::<8>(sum));
3455 let sum = _mm_add_epi16(sum, _mm_srli_si128::<4>(sum));
3456 let sum = _mm_add_epi16(sum, _mm_srli_si128::<2>(sum));
3457 let lanes: [u16; 8usize] = crate::transmute::checked_transmute_copy(&sum);
3458 lanes[0]
3459 }
3460 );
3461 kernel(self, a)
3462 }
3463 #[inline(always)]
3464 fn reduce_product_u16x8(self, a: u16x8<Self>) -> u16 {
3465 crate::kernel!(
3466 #[inline(always)]
3467 fn kernel(token: Avx2, a: u16x8<Avx2>) -> u16 {
3468 let product: __m128i = a.into();
3469 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b11_10_11_10>(product));
3470 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b01_01_01_01>(product));
3471 let product = _mm_mullo_epi16(product, _mm_srli_epi32::<16>(product));
3472 let lanes: [u16; 8usize] = crate::transmute::checked_transmute_copy(&product);
3473 lanes[0]
3474 }
3475 );
3476 kernel(self, a)
3477 }
3478 #[inline(always)]
3479 fn max_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3480 crate::kernel!(
3481 #[inline(always)]
3482 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
3483 _mm_max_epu16(a.into(), b.into()).simd_into(token)
3484 }
3485 );
3486 kernel(self, a, b)
3487 }
3488 #[inline(always)]
3489 fn min_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3490 crate::kernel!(
3491 #[inline(always)]
3492 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
3493 _mm_min_epu16(a.into(), b.into()).simd_into(token)
3494 }
3495 );
3496 kernel(self, a, b)
3497 }
3498 #[inline(always)]
3499 fn clamp_u16x8(self, a: u16x8<Self>, b: u16x8<Self>, c: u16x8<Self>) -> u16x8<Self> {
3500 debug_assert!(
3501 b.simd_le(c).all_true(),
3502 "clamp requires min <= max for all elements"
3503 );
3504 a.max(b).min(c)
3505 }
3506 #[inline(always)]
3507 fn simd_eq_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
3508 crate::kernel!(
3509 #[inline(always)]
3510 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> mask16x8<Avx2> {
3511 _mm_cmpeq_epi16(a.into(), b.into()).simd_into(token)
3512 }
3513 );
3514 kernel(self, a, b)
3515 }
3516 #[inline(always)]
3517 fn simd_lt_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
3518 crate::kernel!(
3519 #[inline(always)]
3520 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> mask16x8<Avx2> {
3521 {
3522 let sign_bit = _mm_set1_epi16(0x8000u16.cast_signed());
3523 let lhs_signed = _mm_xor_si128(b.into(), sign_bit);
3524 let rhs_signed = _mm_xor_si128(a.into(), sign_bit);
3525 _mm_cmpgt_epi16(lhs_signed, rhs_signed)
3526 }
3527 .simd_into(token)
3528 }
3529 );
3530 kernel(self, a, b)
3531 }
3532 #[inline(always)]
3533 fn simd_le_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
3534 crate::kernel!(
3535 #[inline(always)]
3536 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> mask16x8<Avx2> {
3537 _mm_cmpeq_epi16(_mm_min_epu16(a.into(), b.into()), a.into()).simd_into(token)
3538 }
3539 );
3540 kernel(self, a, b)
3541 }
3542 #[inline(always)]
3543 fn zip_low_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3544 crate::kernel!(
3545 #[inline(always)]
3546 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
3547 _mm_unpacklo_epi16(a.into(), b.into()).simd_into(token)
3548 }
3549 );
3550 kernel(self, a, b)
3551 }
3552 #[inline(always)]
3553 fn zip_high_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3554 crate::kernel!(
3555 #[inline(always)]
3556 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
3557 _mm_unpackhi_epi16(a.into(), b.into()).simd_into(token)
3558 }
3559 );
3560 kernel(self, a, b)
3561 }
3562 #[inline(always)]
3563 fn unzip_low_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3564 crate::kernel!(
3565 #[inline(always)]
3566 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
3567 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
3568 let t1 = _mm_shuffle_epi8(a.into(), mask);
3569 let t2 = _mm_shuffle_epi8(b.into(), mask);
3570 _mm_unpacklo_epi64(t1, t2).simd_into(token)
3571 }
3572 );
3573 kernel(self, a, b)
3574 }
3575 #[inline(always)]
3576 fn unzip_high_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3577 crate::kernel!(
3578 #[inline(always)]
3579 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
3580 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
3581 let t1 = _mm_shuffle_epi8(a.into(), mask);
3582 let t2 = _mm_shuffle_epi8(b.into(), mask);
3583 _mm_unpackhi_epi64(t1, t2).simd_into(token)
3584 }
3585 );
3586 kernel(self, a, b)
3587 }
3588 #[inline(always)]
3589 fn interleave_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u16x8<Self>, u16x8<Self>) {
3590 (self.zip_low_u16x8(a, b), self.zip_high_u16x8(a, b))
3591 }
3592 #[inline(always)]
3593 fn deinterleave_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u16x8<Self>, u16x8<Self>) {
3594 crate::kernel!(
3595 #[inline(always)]
3596 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> (u16x8<Avx2>, u16x8<Avx2>) {
3597 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
3598 let a = _mm_shuffle_epi8(a.into(), mask);
3599 let b = _mm_shuffle_epi8(b.into(), mask);
3600 (
3601 _mm_unpacklo_epi64(a, b).simd_into(token),
3602 _mm_unpackhi_epi64(a, b).simd_into(token),
3603 )
3604 }
3605 );
3606 kernel(self, a, b)
3607 }
3608 #[inline(always)]
3609 fn select_u16x8(self, a: mask16x8<Self>, b: u16x8<Self>, c: u16x8<Self>) -> u16x8<Self> {
3610 crate::kernel!(
3611 #[inline(always)]
3612 fn kernel(
3613 token: Avx2,
3614 a: mask16x8<Avx2>,
3615 b: u16x8<Avx2>,
3616 c: u16x8<Avx2>,
3617 ) -> u16x8<Avx2> {
3618 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
3619 }
3620 );
3621 kernel(self, a, b, c)
3622 }
3623 #[inline(always)]
3624 fn combine_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x16<Self> {
3625 crate::kernel!(
3626 #[inline(always)]
3627 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x16<Avx2> {
3628 _mm256_setr_m128i(a.into(), b.into()).simd_into(token)
3629 }
3630 );
3631 kernel(self, a, b)
3632 }
3633 #[inline(always)]
3634 fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8<Self>; 4usize] {
3635 crate::kernel!(
3636 #[inline(always)]
3637 fn kernel(token: Avx2, src: &[u16; 32usize]) -> [u16x8<Avx2>; 4usize] {
3638 let (chunks, []) = src.as_chunks::<8usize>() else {
3639 unreachable!()
3640 };
3641 let v0: __m128i =
3642 crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[0]);
3643 let v1: __m128i =
3644 crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[1]);
3645 let v2: __m128i =
3646 crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[2]);
3647 let v3: __m128i =
3648 crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[3]);
3649 let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15);
3650 let v0 = _mm_shuffle_epi8(v0, mask);
3651 let v1 = _mm_shuffle_epi8(v1, mask);
3652 let v2 = _mm_shuffle_epi8(v2, mask);
3653 let v3 = _mm_shuffle_epi8(v3, mask);
3654 let tmp0 = _mm_unpacklo_epi32(v0, v1);
3655 let tmp1 = _mm_unpackhi_epi32(v0, v1);
3656 let tmp2 = _mm_unpacklo_epi32(v2, v3);
3657 let tmp3 = _mm_unpackhi_epi32(v2, v3);
3658 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
3659 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
3660 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
3661 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
3662 [
3663 out0.simd_into(token),
3664 out1.simd_into(token),
3665 out2.simd_into(token),
3666 out3.simd_into(token),
3667 ]
3668 }
3669 );
3670 kernel(self, src)
3671 }
3672 #[inline(always)]
3673 fn store_four_interleaved_u16x8(
3674 self,
3675 vectors: [u16x8<Self>; 4usize],
3676 dest: &mut [u16; 32usize],
3677 ) -> () {
3678 crate::kernel!(
3679 #[inline(always)]
3680 fn kernel(
3681 token: Avx2,
3682 vectors: [u16x8<Avx2>; 4usize],
3683 dest: &mut [u16; 32usize],
3684 ) -> () {
3685 let _ = token;
3686 let v0: __m128i = vectors[0].into();
3687 let v1: __m128i = vectors[1].into();
3688 let v2: __m128i = vectors[2].into();
3689 let v3: __m128i = vectors[3].into();
3690 let tmp0 = _mm_unpacklo_epi32(v0, v1);
3691 let tmp1 = _mm_unpackhi_epi32(v0, v1);
3692 let tmp2 = _mm_unpacklo_epi32(v2, v3);
3693 let tmp3 = _mm_unpackhi_epi32(v2, v3);
3694 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
3695 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
3696 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
3697 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
3698 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
3699 let out0 = _mm_shuffle_epi8(out0, mask);
3700 let out1 = _mm_shuffle_epi8(out1, mask);
3701 let out2 = _mm_shuffle_epi8(out2, mask);
3702 let out3 = _mm_shuffle_epi8(out3, mask);
3703 let (chunks, []) = dest.as_chunks_mut::<8usize>() else {
3704 unreachable!()
3705 };
3706 crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>(
3707 out0,
3708 &mut chunks[0],
3709 );
3710 crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>(
3711 out1,
3712 &mut chunks[1],
3713 );
3714 crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>(
3715 out2,
3716 &mut chunks[2],
3717 );
3718 crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>(
3719 out3,
3720 &mut chunks[3],
3721 );
3722 }
3723 );
3724 kernel(self, vectors, dest);
3725 }
3726 #[inline(always)]
3727 fn widen_u16x8(self, a: u16x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
3728 crate::kernel!(
3729 #[inline(always)]
3730 fn kernel(token: Avx2, a: u16x8<Avx2>) -> (u32x4<Avx2>, u32x4<Avx2>) {
3731 let raw = a.into();
3732 (
3733 _mm_cvtepu16_epi32(raw).simd_into(token),
3734 _mm_cvtepu16_epi32(_mm_srli_si128::<8>(raw)).simd_into(token),
3735 )
3736 }
3737 );
3738 kernel(self, a)
3739 }
3740 #[inline(always)]
3741 fn widening_mul_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
3742 crate::kernel!(
3743 #[inline(always)]
3744 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> (u32x4<Avx2>, u32x4<Avx2>) {
3745 let a = a.into();
3746 let b = b.into();
3747 let low = _mm_mullo_epi16(a, b);
3748 let high = _mm_mulhi_epu16(a, b);
3749 let result_low = _mm_unpacklo_epi16(low, high);
3750 let result_high = _mm_unpackhi_epi16(low, high);
3751 (result_low.simd_into(token), result_high.simd_into(token))
3752 }
3753 );
3754 kernel(self, a, b)
3755 }
3756 #[inline(always)]
3757 fn widening_add_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
3758 let (a0, a1) = self.widen_u16x8(a);
3759 let (b0, b1) = self.widen_u16x8(b);
3760 (self.add_u32x4(a0, b0), self.add_u32x4(a1, b1))
3761 }
3762 #[inline(always)]
3763 fn pairwise_widening_add_u16x8(self, a: u16x8<Self>) -> u32x4<Self> {
3764 crate::kernel!(
3765 #[inline(always)]
3766 fn kernel(token: Avx2, a: u16x8<Avx2>) -> u32x4<Avx2> {
3767 ({
3768 let a = a.into();
3769 _mm_add_epi32(
3770 _mm_and_si128(a, _mm_set1_epi32(65535)),
3771 _mm_srli_epi32::<16>(a),
3772 )
3773 })
3774 .simd_into(token)
3775 }
3776 );
3777 kernel(self, a)
3778 }
3779 #[inline(always)]
3780 fn pairwise_dot_product_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u32x4<Self> {
3781 let (low, high) = self.widening_mul_u16x8(a, b);
3782 let (even, odd) = self.deinterleave_u32x4(low, high);
3783 self.add_u32x4(even, odd)
3784 }
3785 #[inline(always)]
3786 fn narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
3787 crate::kernel!(
3788 #[inline(always)]
3789 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u8x16<Avx2> {
3790 let mask = _mm_set1_epi16(0xff);
3791 _mm_packus_epi16(_mm_and_si128(a.into(), mask), _mm_and_si128(b.into(), mask))
3792 .simd_into(token)
3793 }
3794 );
3795 kernel(self, a, b)
3796 }
3797 #[inline(always)]
3798 fn saturating_narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
3799 crate::kernel!(
3800 #[inline(always)]
3801 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u8x16<Avx2> {
3802 {
3803 let max = _mm_set1_epi16(i16::MAX);
3804 _mm_packus_epi16(_mm_min_epu16(a.into(), max), _mm_min_epu16(b.into(), max))
3805 }
3806 .simd_into(token)
3807 }
3808 );
3809 kernel(self, a, b)
3810 }
3811 #[inline(always)]
3812 fn relaxed_narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
3813 crate::kernel!(
3814 #[inline(always)]
3815 fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u8x16<Avx2> {
3816 debug_assert!(
3817 a.as_slice()
3818 .iter()
3819 .chain(b.as_slice())
3820 .all(|&value| { value >= u8::MIN as u16 && value <= u8::MAX as u16 }),
3821 "relaxed_narrow inputs must fit in the destination type",
3822 );
3823 _mm_packus_epi16(a.into(), b.into()).simd_into(token)
3824 }
3825 );
3826 kernel(self, a, b)
3827 }
3828 #[inline(always)]
3829 fn splat_mask16x8(self, val: bool) -> mask16x8<Self> {
3830 crate::kernel!(
3831 #[inline(always)]
3832 fn kernel(token: Avx2, val: bool) -> mask16x8<Avx2> {
3833 let val: i16 = if val { !0 } else { 0 };
3834 _mm_set1_epi16(val).simd_into(token)
3835 }
3836 );
3837 kernel(self, val)
3838 }
3839 #[inline(always)]
3840 fn from_bitmask_mask16x8(self, bits: u64) -> mask16x8<Self> {
3841 crate::kernel!(
3842 #[inline(always)]
3843 fn kernel(token: Avx2, bits: u64) -> mask16x8<Avx2> {
3844 {
3845 let bit_lanes = _mm_set1_epi16(bits as i16);
3846 let bit_mask = _mm_setr_epi16(1, 2, 4, 8, 16, 32, 64, 128);
3847 _mm_cmpeq_epi16(_mm_and_si128(bit_lanes, bit_mask), bit_mask)
3848 }
3849 .simd_into(token)
3850 }
3851 );
3852 kernel(self, bits)
3853 }
3854 #[inline(always)]
3855 fn to_bitmask_mask16x8(self, a: mask16x8<Self>) -> u64 {
3856 crate::kernel!(
3857 #[inline(always)]
3858 fn kernel(token: Avx2, a: mask16x8<Avx2>) -> u64 {
3859 {
3860 let packed = _mm_packs_epi16(a.into(), a.into());
3861 _mm_movemask_epi8(packed) as u8 as u64
3862 }
3863 }
3864 );
3865 kernel(self, a)
3866 }
3867 #[inline(always)]
3868 fn from_vector_mask16x8(self, vector: i16x8<Self>) -> mask16x8<Self> {
3869 mask16x8 {
3870 val: vector.val,
3871 simd: self,
3872 }
3873 }
3874 #[inline(always)]
3875 fn to_vector_mask16x8(self, a: mask16x8<Self>) -> i16x8<Self> {
3876 i16x8 {
3877 val: a.val,
3878 simd: self,
3879 }
3880 }
3881 #[inline(always)]
3882 fn set_mask16x8(self, a: &mut mask16x8<Self>, index: usize, value: bool) -> () {
3883 assert!(
3884 index < 8usize,
3885 "mask lane index {index} is out of bounds for {} lanes",
3886 8usize
3887 );
3888 let mut lanes: [i16; 8usize] = (*a).into();
3889 lanes[index] = if value { !0 } else { 0 };
3890 *a = lanes.simd_into(self);
3891 }
3892 #[inline(always)]
3893 fn rotate_elements_left_mask16x8<const OFFSET: usize>(
3894 self,
3895 a: mask16x8<Self>,
3896 ) -> mask16x8<Self> {
3897 let int = i16x8 {
3898 val: crate::transmute::checked_transmute_copy(&a.val),
3899 simd: self,
3900 };
3901 let rotated = int.rotate_elements_left::<OFFSET>();
3902 mask16x8 {
3903 val: crate::transmute::checked_transmute_copy(&rotated.val),
3904 simd: self,
3905 }
3906 }
3907 #[inline(always)]
3908 fn rotate_elements_right_mask16x8<const OFFSET: usize>(
3909 self,
3910 a: mask16x8<Self>,
3911 ) -> mask16x8<Self> {
3912 let int = i16x8 {
3913 val: crate::transmute::checked_transmute_copy(&a.val),
3914 simd: self,
3915 };
3916 let rotated = int.rotate_elements_right::<OFFSET>();
3917 mask16x8 {
3918 val: crate::transmute::checked_transmute_copy(&rotated.val),
3919 simd: self,
3920 }
3921 }
3922 #[inline(always)]
3923 fn and_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
3924 crate::kernel!(
3925 #[inline(always)]
3926 fn kernel(token: Avx2, a: mask16x8<Avx2>, b: mask16x8<Avx2>) -> mask16x8<Avx2> {
3927 _mm_and_si128(a.into(), b.into()).simd_into(token)
3928 }
3929 );
3930 kernel(self, a, b)
3931 }
3932 #[inline(always)]
3933 fn or_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
3934 crate::kernel!(
3935 #[inline(always)]
3936 fn kernel(token: Avx2, a: mask16x8<Avx2>, b: mask16x8<Avx2>) -> mask16x8<Avx2> {
3937 _mm_or_si128(a.into(), b.into()).simd_into(token)
3938 }
3939 );
3940 kernel(self, a, b)
3941 }
3942 #[inline(always)]
3943 fn xor_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
3944 crate::kernel!(
3945 #[inline(always)]
3946 fn kernel(token: Avx2, a: mask16x8<Avx2>, b: mask16x8<Avx2>) -> mask16x8<Avx2> {
3947 _mm_xor_si128(a.into(), b.into()).simd_into(token)
3948 }
3949 );
3950 kernel(self, a, b)
3951 }
3952 #[inline(always)]
3953 fn not_mask16x8(self, a: mask16x8<Self>) -> mask16x8<Self> {
3954 self.xor_mask16x8(a, self.splat_mask16x8(true))
3955 }
3956 #[inline(always)]
3957 fn reverse_mask16x8(self, a: mask16x8<Self>) -> mask16x8<Self> {
3958 let lanes = i16x8 {
3959 val: crate::transmute::checked_transmute_copy(&a.val),
3960 simd: self,
3961 };
3962 let reversed = self.reverse_i16x8(lanes);
3963 mask16x8 {
3964 val: crate::transmute::checked_transmute_copy(&reversed.val),
3965 simd: self,
3966 }
3967 }
3968 #[inline(always)]
3969 fn select_mask16x8(
3970 self,
3971 a: mask16x8<Self>,
3972 b: mask16x8<Self>,
3973 c: mask16x8<Self>,
3974 ) -> mask16x8<Self> {
3975 crate::kernel!(
3976 #[inline(always)]
3977 fn kernel(
3978 token: Avx2,
3979 a: mask16x8<Avx2>,
3980 b: mask16x8<Avx2>,
3981 c: mask16x8<Avx2>,
3982 ) -> mask16x8<Avx2> {
3983 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
3984 }
3985 );
3986 kernel(self, a, b, c)
3987 }
3988 #[inline(always)]
3989 fn simd_eq_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
3990 crate::kernel!(
3991 #[inline(always)]
3992 fn kernel(token: Avx2, a: mask16x8<Avx2>, b: mask16x8<Avx2>) -> mask16x8<Avx2> {
3993 _mm_cmpeq_epi16(a.into(), b.into()).simd_into(token)
3994 }
3995 );
3996 kernel(self, a, b)
3997 }
3998 #[inline(always)]
3999 fn any_true_mask16x8(self, a: mask16x8<Self>) -> bool {
4000 crate::kernel!(
4001 #[inline(always)]
4002 fn kernel(token: Avx2, a: mask16x8<Avx2>) -> bool {
4003 _mm_movemask_epi8(a.into()) as u32 != 0
4004 }
4005 );
4006 kernel(self, a)
4007 }
4008 #[inline(always)]
4009 fn all_true_mask16x8(self, a: mask16x8<Self>) -> bool {
4010 crate::kernel!(
4011 #[inline(always)]
4012 fn kernel(token: Avx2, a: mask16x8<Avx2>) -> bool {
4013 _mm_movemask_epi8(a.into()) as u32 == 0xffff
4014 }
4015 );
4016 kernel(self, a)
4017 }
4018 #[inline(always)]
4019 fn any_false_mask16x8(self, a: mask16x8<Self>) -> bool {
4020 crate::kernel!(
4021 #[inline(always)]
4022 fn kernel(token: Avx2, a: mask16x8<Avx2>) -> bool {
4023 _mm_movemask_epi8(a.into()) as u32 != 0xffff
4024 }
4025 );
4026 kernel(self, a)
4027 }
4028 #[inline(always)]
4029 fn all_false_mask16x8(self, a: mask16x8<Self>) -> bool {
4030 crate::kernel!(
4031 #[inline(always)]
4032 fn kernel(token: Avx2, a: mask16x8<Avx2>) -> bool {
4033 _mm_movemask_epi8(a.into()) as u32 == 0
4034 }
4035 );
4036 kernel(self, a)
4037 }
4038 #[inline(always)]
4039 fn combine_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x16<Self> {
4040 crate::kernel!(
4041 #[inline(always)]
4042 fn kernel(token: Avx2, a: mask16x8<Avx2>, b: mask16x8<Avx2>) -> mask16x16<Avx2> {
4043 _mm256_setr_m128i(a.into(), b.into()).simd_into(token)
4044 }
4045 );
4046 kernel(self, a, b)
4047 }
4048 #[inline(always)]
4049 fn widen_mask16x8(self, a: mask16x8<Self>) -> (mask32x4<Self>, mask32x4<Self>) {
4050 crate::kernel!(
4051 #[inline(always)]
4052 fn kernel(token: Avx2, a: mask16x8<Avx2>) -> (mask32x4<Avx2>, mask32x4<Avx2>) {
4053 let raw = a.into();
4054 (
4055 _mm_unpacklo_epi16(raw, raw).simd_into(token),
4056 _mm_unpackhi_epi16(raw, raw).simd_into(token),
4057 )
4058 }
4059 );
4060 kernel(self, a)
4061 }
4062 #[inline(always)]
4063 fn narrow_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask8x16<Self> {
4064 crate::kernel!(
4065 #[inline(always)]
4066 fn kernel(token: Avx2, a: mask16x8<Avx2>, b: mask16x8<Avx2>) -> mask8x16<Avx2> {
4067 _mm_packs_epi16(a.into(), b.into()).simd_into(token)
4068 }
4069 );
4070 kernel(self, a, b)
4071 }
4072 #[inline(always)]
4073 fn abs_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4074 crate::kernel!(
4075 #[inline(always)]
4076 fn kernel(token: Avx2, a: i32x4<Avx2>) -> i32x4<Avx2> {
4077 _mm_abs_epi32(a.into()).simd_into(token)
4078 }
4079 );
4080 kernel(self, a)
4081 }
4082 #[inline(always)]
4083 fn splat_i32x4(self, val: i32) -> i32x4<Self> {
4084 crate::kernel!(
4085 #[inline(always)]
4086 fn kernel(token: Avx2, val: i32) -> i32x4<Avx2> {
4087 _mm_set1_epi32(val).simd_into(token)
4088 }
4089 );
4090 kernel(self, val)
4091 }
4092 #[inline(always)]
4093 fn reverse_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4094 let indices: u8x16<Self> =
4095 [12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3].simd_into(self);
4096 self.swizzle_dyn_i32x4(a, indices)
4097 }
4098 #[inline(always)]
4099 fn slide_i32x4<const SHIFT: usize>(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4100 if SHIFT >= 4usize {
4101 return b;
4102 }
4103 let result = dyn_alignr_128(
4104 self,
4105 Bytes::to_bytes(b).val.0,
4106 Bytes::to_bytes(a).val.0,
4107 SHIFT * 4usize,
4108 );
4109 Bytes::from_bytes(u8x16 {
4110 val: crate::support::Aligned128(result),
4111 simd: self,
4112 })
4113 }
4114 #[inline(always)]
4115 fn compress_i32x4(self, values: i32x4<Self>, mask: mask32x4<Self>) -> i32x4<Self> {
4116 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
4117 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0);
4118 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
4119 }
4120 #[inline(always)]
4121 fn compress_merge_i32x4(
4122 self,
4123 values: i32x4<Self>,
4124 mask: mask32x4<Self>,
4125 merge: i32x4<Self>,
4126 ) -> i32x4<Self> {
4127 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
4128 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0);
4129 let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control);
4130 let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0));
4131 Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed))
4132 }
4133 #[inline(always)]
4134 fn expand_i32x4(self, values: i32x4<Self>, mask: mask32x4<Self>) -> i32x4<Self> {
4135 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
4136 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0);
4137 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
4138 }
4139 #[inline(always)]
4140 fn expand_merge_i32x4(
4141 self,
4142 values: i32x4<Self>,
4143 mask: mask32x4<Self>,
4144 merge: i32x4<Self>,
4145 ) -> i32x4<Self> {
4146 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
4147 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0);
4148 let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control));
4149 self.select_i32x4(mask, expanded, merge)
4150 }
4151 #[inline(always)]
4152 fn count_ones_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4153 crate::kernel!(
4154 #[inline(always)]
4155 fn kernel(token: Avx2, a: i32x4<Avx2>) -> i32x4<Avx2> {
4156 let value = a.into();
4157 let nibble_mask = _mm_set1_epi8(0x0f);
4158 let lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4);
4159 let low = _mm_and_si128(value, nibble_mask);
4160 let high = _mm_and_si128(_mm_srli_epi16::<4>(value), nibble_mask);
4161 let byte_counts = _mm_add_epi8(
4162 _mm_shuffle_epi8(lookup, low),
4163 _mm_shuffle_epi8(lookup, high),
4164 );
4165 _mm_madd_epi16(
4166 _mm_maddubs_epi16(byte_counts, _mm_set1_epi8(1)),
4167 _mm_set1_epi16(1),
4168 )
4169 .simd_into(token)
4170 }
4171 );
4172 kernel(self, a)
4173 }
4174 #[inline(always)]
4175 fn count_zeros_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4176 self.count_ones_i32x4(self.not_i32x4(a))
4177 }
4178 #[inline(always)]
4179 fn add_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4180 crate::kernel!(
4181 #[inline(always)]
4182 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
4183 _mm_add_epi32(a.into(), b.into()).simd_into(token)
4184 }
4185 );
4186 kernel(self, a, b)
4187 }
4188 #[inline(always)]
4189 fn saturating_add_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4190 crate::kernel!(
4191 #[inline(always)]
4192 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
4193 let a = a.into();
4194 let b = b.into();
4195 let wrapped = _mm_add_epi32(a, b);
4196 let overflow = _mm_xor_si128(_mm_cmpgt_epi32(a, wrapped), b);
4197 let bound = _mm_xor_si128(_mm_srai_epi32::<31>(a), _mm_set1_epi32(i32::MAX));
4198 let result = _mm_blendv_ps(
4199 _mm_castsi128_ps(wrapped),
4200 _mm_castsi128_ps(bound),
4201 _mm_castsi128_ps(overflow),
4202 );
4203 _mm_castps_si128(result).simd_into(token)
4204 }
4205 );
4206 kernel(self, a, b)
4207 }
4208 #[inline(always)]
4209 fn sub_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4210 crate::kernel!(
4211 #[inline(always)]
4212 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
4213 _mm_sub_epi32(a.into(), b.into()).simd_into(token)
4214 }
4215 );
4216 kernel(self, a, b)
4217 }
4218 #[inline(always)]
4219 fn saturating_sub_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4220 crate::kernel!(
4221 #[inline(always)]
4222 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
4223 let a = a.into();
4224 let b = b.into();
4225 let wrapped = _mm_sub_epi32(a, b);
4226 let overflow = _mm_xor_si128(_mm_cmpgt_epi32(wrapped, a), b);
4227 let bound = _mm_xor_si128(_mm_srai_epi32::<31>(a), _mm_set1_epi32(i32::MAX));
4228 let result = _mm_blendv_ps(
4229 _mm_castsi128_ps(wrapped),
4230 _mm_castsi128_ps(bound),
4231 _mm_castsi128_ps(overflow),
4232 );
4233 _mm_castps_si128(result).simd_into(token)
4234 }
4235 );
4236 kernel(self, a, b)
4237 }
4238 #[inline(always)]
4239 fn mul_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4240 crate::kernel!(
4241 #[inline(always)]
4242 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
4243 _mm_mullo_epi32(a.into(), b.into()).simd_into(token)
4244 }
4245 );
4246 kernel(self, a, b)
4247 }
4248 #[inline(always)]
4249 fn and_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4250 crate::kernel!(
4251 #[inline(always)]
4252 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
4253 _mm_and_si128(a.into(), b.into()).simd_into(token)
4254 }
4255 );
4256 kernel(self, a, b)
4257 }
4258 #[inline(always)]
4259 fn or_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4260 crate::kernel!(
4261 #[inline(always)]
4262 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
4263 _mm_or_si128(a.into(), b.into()).simd_into(token)
4264 }
4265 );
4266 kernel(self, a, b)
4267 }
4268 #[inline(always)]
4269 fn xor_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4270 crate::kernel!(
4271 #[inline(always)]
4272 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
4273 _mm_xor_si128(a.into(), b.into()).simd_into(token)
4274 }
4275 );
4276 kernel(self, a, b)
4277 }
4278 #[inline(always)]
4279 fn not_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4280 a ^ !0
4281 }
4282 #[inline(always)]
4283 fn shl_i32x4(self, a: i32x4<Self>, shift: u32) -> i32x4<Self> {
4284 crate::kernel!(
4285 #[inline(always)]
4286 fn kernel(token: Avx2, a: i32x4<Avx2>, shift: u32) -> i32x4<Avx2> {
4287 _mm_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
4288 }
4289 );
4290 kernel(self, a, shift)
4291 }
4292 #[inline(always)]
4293 fn shlv_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4294 crate::kernel!(
4295 #[inline(always)]
4296 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
4297 _mm_sllv_epi32(a.into(), b.into()).simd_into(token)
4298 }
4299 );
4300 kernel(self, a, b)
4301 }
4302 #[inline(always)]
4303 fn shr_i32x4(self, a: i32x4<Self>, shift: u32) -> i32x4<Self> {
4304 crate::kernel!(
4305 #[inline(always)]
4306 fn kernel(token: Avx2, a: i32x4<Avx2>, shift: u32) -> i32x4<Avx2> {
4307 _mm_sra_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
4308 }
4309 );
4310 kernel(self, a, shift)
4311 }
4312 #[inline(always)]
4313 fn shrv_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4314 crate::kernel!(
4315 #[inline(always)]
4316 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
4317 _mm_srav_epi32(a.into(), b.into()).simd_into(token)
4318 }
4319 );
4320 kernel(self, a, b)
4321 }
4322 #[inline(always)]
4323 fn reduce_max_i32x4(self, a: i32x4<Self>) -> i32 {
4324 crate::kernel!(
4325 #[inline(always)]
4326 fn kernel(token: Avx2, a: i32x4<Avx2>) -> i32 {
4327 let reduced: __m128i = a.into();
4328 let shifted = _mm_srli_si128::<8>(reduced);
4329 let reduced = { _mm_max_epi32(reduced, shifted) };
4330 let shifted = _mm_srli_si128::<4>(reduced);
4331 let reduced = { _mm_max_epi32(reduced, shifted) };
4332 {
4333 let lanes: [i32; 4usize] = crate::transmute::checked_transmute_copy(&reduced);
4334 lanes[0]
4335 }
4336 }
4337 );
4338 kernel(self, a)
4339 }
4340 #[inline(always)]
4341 fn reduce_min_i32x4(self, a: i32x4<Self>) -> i32 {
4342 crate::kernel!(
4343 #[inline(always)]
4344 fn kernel(token: Avx2, a: i32x4<Avx2>) -> i32 {
4345 let reduced: __m128i = a.into();
4346 let shifted = _mm_srli_si128::<8>(reduced);
4347 let reduced = { _mm_min_epi32(reduced, shifted) };
4348 let shifted = _mm_srli_si128::<4>(reduced);
4349 let reduced = { _mm_min_epi32(reduced, shifted) };
4350 {
4351 let lanes: [i32; 4usize] = crate::transmute::checked_transmute_copy(&reduced);
4352 lanes[0]
4353 }
4354 }
4355 );
4356 kernel(self, a)
4357 }
4358 #[inline(always)]
4359 fn reduce_sum_i32x4(self, a: i32x4<Self>) -> i32 {
4360 crate::kernel!(
4361 #[inline(always)]
4362 fn kernel(token: Avx2, a: i32x4<Avx2>) -> i32 {
4363 let sum: __m128i = a.into();
4364 let sum = _mm_add_epi32(sum, _mm_srli_si128::<8>(sum));
4365 let sum = _mm_add_epi32(sum, _mm_srli_si128::<4>(sum));
4366 let lanes: [i32; 4usize] = crate::transmute::checked_transmute_copy(&sum);
4367 lanes[0]
4368 }
4369 );
4370 kernel(self, a)
4371 }
4372 #[inline(always)]
4373 fn reduce_product_i32x4(self, a: i32x4<Self>) -> i32 {
4374 let mul_level_0: [i32; 2usize] = [
4375 a[0usize].wrapping_mul(a[1usize]),
4376 a[2usize].wrapping_mul(a[3usize]),
4377 ];
4378 let mul_level_1: [i32; 1usize] = [mul_level_0[0usize].wrapping_mul(mul_level_0[1usize])];
4379 mul_level_1[0]
4380 }
4381 #[inline(always)]
4382 fn max_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4383 crate::kernel!(
4384 #[inline(always)]
4385 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
4386 _mm_max_epi32(a.into(), b.into()).simd_into(token)
4387 }
4388 );
4389 kernel(self, a, b)
4390 }
4391 #[inline(always)]
4392 fn min_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4393 crate::kernel!(
4394 #[inline(always)]
4395 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
4396 _mm_min_epi32(a.into(), b.into()).simd_into(token)
4397 }
4398 );
4399 kernel(self, a, b)
4400 }
4401 #[inline(always)]
4402 fn clamp_i32x4(self, a: i32x4<Self>, b: i32x4<Self>, c: i32x4<Self>) -> i32x4<Self> {
4403 debug_assert!(
4404 b.simd_le(c).all_true(),
4405 "clamp requires min <= max for all elements"
4406 );
4407 a.max(b).min(c)
4408 }
4409 #[inline(always)]
4410 fn simd_eq_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
4411 crate::kernel!(
4412 #[inline(always)]
4413 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> mask32x4<Avx2> {
4414 _mm_cmpeq_epi32(a.into(), b.into()).simd_into(token)
4415 }
4416 );
4417 kernel(self, a, b)
4418 }
4419 #[inline(always)]
4420 fn simd_lt_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
4421 crate::kernel!(
4422 #[inline(always)]
4423 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> mask32x4<Avx2> {
4424 _mm_cmpgt_epi32(b.into(), a.into()).simd_into(token)
4425 }
4426 );
4427 kernel(self, a, b)
4428 }
4429 #[inline(always)]
4430 fn simd_le_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
4431 crate::kernel!(
4432 #[inline(always)]
4433 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> mask32x4<Avx2> {
4434 _mm_cmpeq_epi32(_mm_min_epi32(a.into(), b.into()), a.into()).simd_into(token)
4435 }
4436 );
4437 kernel(self, a, b)
4438 }
4439 #[inline(always)]
4440 fn zip_low_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4441 crate::kernel!(
4442 #[inline(always)]
4443 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
4444 _mm_unpacklo_epi32(a.into(), b.into()).simd_into(token)
4445 }
4446 );
4447 kernel(self, a, b)
4448 }
4449 #[inline(always)]
4450 fn zip_high_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4451 crate::kernel!(
4452 #[inline(always)]
4453 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
4454 _mm_unpackhi_epi32(a.into(), b.into()).simd_into(token)
4455 }
4456 );
4457 kernel(self, a, b)
4458 }
4459 #[inline(always)]
4460 fn unzip_low_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4461 crate::kernel!(
4462 #[inline(always)]
4463 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
4464 let t1 = _mm_shuffle_epi32::<0b11_01_10_00>(a.into());
4465 let t2 = _mm_shuffle_epi32::<0b11_01_10_00>(b.into());
4466 _mm_unpacklo_epi64(t1, t2).simd_into(token)
4467 }
4468 );
4469 kernel(self, a, b)
4470 }
4471 #[inline(always)]
4472 fn unzip_high_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4473 crate::kernel!(
4474 #[inline(always)]
4475 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
4476 let t1 = _mm_shuffle_epi32::<0b11_01_10_00>(a.into());
4477 let t2 = _mm_shuffle_epi32::<0b11_01_10_00>(b.into());
4478 _mm_unpackhi_epi64(t1, t2).simd_into(token)
4479 }
4480 );
4481 kernel(self, a, b)
4482 }
4483 #[inline(always)]
4484 fn interleave_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i32x4<Self>, i32x4<Self>) {
4485 (self.zip_low_i32x4(a, b), self.zip_high_i32x4(a, b))
4486 }
4487 #[inline(always)]
4488 fn deinterleave_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i32x4<Self>, i32x4<Self>) {
4489 (self.unzip_low_i32x4(a, b), self.unzip_high_i32x4(a, b))
4490 }
4491 #[inline(always)]
4492 fn select_i32x4(self, a: mask32x4<Self>, b: i32x4<Self>, c: i32x4<Self>) -> i32x4<Self> {
4493 crate::kernel!(
4494 #[inline(always)]
4495 fn kernel(
4496 token: Avx2,
4497 a: mask32x4<Avx2>,
4498 b: i32x4<Avx2>,
4499 c: i32x4<Avx2>,
4500 ) -> i32x4<Avx2> {
4501 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
4502 }
4503 );
4504 kernel(self, a, b, c)
4505 }
4506 #[inline(always)]
4507 fn combine_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x8<Self> {
4508 crate::kernel!(
4509 #[inline(always)]
4510 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x8<Avx2> {
4511 _mm256_setr_m128i(a.into(), b.into()).simd_into(token)
4512 }
4513 );
4514 kernel(self, a, b)
4515 }
4516 #[inline(always)]
4517 fn neg_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4518 crate::kernel!(
4519 #[inline(always)]
4520 fn kernel(token: Avx2, a: i32x4<Avx2>) -> i32x4<Avx2> {
4521 _mm_sub_epi32(_mm_setzero_si128(), a.into()).simd_into(token)
4522 }
4523 );
4524 kernel(self, a)
4525 }
4526 #[inline(always)]
4527 fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4<Self>; 4usize] {
4528 crate::kernel!(
4529 #[inline(always)]
4530 fn kernel(token: Avx2, src: &[i32; 16usize]) -> [i32x4<Avx2>; 4usize] {
4531 let (chunks, []) = src.as_chunks::<4usize>() else {
4532 unreachable!()
4533 };
4534 let v0: __m128i =
4535 crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[0]);
4536 let v1: __m128i =
4537 crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[1]);
4538 let v2: __m128i =
4539 crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[2]);
4540 let v3: __m128i =
4541 crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[3]);
4542 let tmp0 = _mm_unpacklo_epi32(v0, v1);
4543 let tmp1 = _mm_unpackhi_epi32(v0, v1);
4544 let tmp2 = _mm_unpacklo_epi32(v2, v3);
4545 let tmp3 = _mm_unpackhi_epi32(v2, v3);
4546 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
4547 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
4548 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
4549 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
4550 [
4551 out0.simd_into(token),
4552 out1.simd_into(token),
4553 out2.simd_into(token),
4554 out3.simd_into(token),
4555 ]
4556 }
4557 );
4558 kernel(self, src)
4559 }
4560 #[inline(always)]
4561 fn store_four_interleaved_i32x4(
4562 self,
4563 vectors: [i32x4<Self>; 4usize],
4564 dest: &mut [i32; 16usize],
4565 ) -> () {
4566 crate::kernel!(
4567 #[inline(always)]
4568 fn kernel(
4569 token: Avx2,
4570 vectors: [i32x4<Avx2>; 4usize],
4571 dest: &mut [i32; 16usize],
4572 ) -> () {
4573 let _ = token;
4574 let v0: __m128i = vectors[0].into();
4575 let v1: __m128i = vectors[1].into();
4576 let v2: __m128i = vectors[2].into();
4577 let v3: __m128i = vectors[3].into();
4578 let tmp0 = _mm_unpacklo_epi32(v0, v1);
4579 let tmp1 = _mm_unpackhi_epi32(v0, v1);
4580 let tmp2 = _mm_unpacklo_epi32(v2, v3);
4581 let tmp3 = _mm_unpackhi_epi32(v2, v3);
4582 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
4583 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
4584 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
4585 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
4586 let (chunks, []) = dest.as_chunks_mut::<4usize>() else {
4587 unreachable!()
4588 };
4589 crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>(
4590 out0,
4591 &mut chunks[0],
4592 );
4593 crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>(
4594 out1,
4595 &mut chunks[1],
4596 );
4597 crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>(
4598 out2,
4599 &mut chunks[2],
4600 );
4601 crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>(
4602 out3,
4603 &mut chunks[3],
4604 );
4605 }
4606 );
4607 kernel(self, vectors, dest);
4608 }
4609 #[inline(always)]
4610 fn widen_i32x4(self, a: i32x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
4611 crate::kernel!(
4612 #[inline(always)]
4613 fn kernel(token: Avx2, a: i32x4<Avx2>) -> (i64x2<Avx2>, i64x2<Avx2>) {
4614 let raw = a.into();
4615 (
4616 _mm_cvtepi32_epi64(raw).simd_into(token),
4617 _mm_cvtepi32_epi64(_mm_srli_si128::<8>(raw)).simd_into(token),
4618 )
4619 }
4620 );
4621 kernel(self, a)
4622 }
4623 #[inline(always)]
4624 fn widening_mul_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
4625 crate::kernel!(
4626 #[inline(always)]
4627 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> (i64x2<Avx2>, i64x2<Avx2>) {
4628 let a = a.into();
4629 let b = b.into();
4630 let low = _mm_mul_epi32(a, b);
4631 let high =
4632 _mm_mul_epi32(_mm_shuffle_epi32::<0xf5>(a), _mm_shuffle_epi32::<0xf5>(b));
4633 let result_low = _mm_unpacklo_epi64(low, high);
4634 let result_high = _mm_unpackhi_epi64(low, high);
4635 (result_low.simd_into(token), result_high.simd_into(token))
4636 }
4637 );
4638 kernel(self, a, b)
4639 }
4640 #[inline(always)]
4641 fn widening_add_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
4642 let (a0, a1) = self.widen_i32x4(a);
4643 let (b0, b1) = self.widen_i32x4(b);
4644 (self.add_i64x2(a0, b0), self.add_i64x2(a1, b1))
4645 }
4646 #[inline(always)]
4647 fn pairwise_widening_add_i32x4(self, a: i32x4<Self>) -> i64x2<Self> {
4648 crate::kernel!(
4649 #[inline(always)]
4650 fn kernel(token: Avx2, a: i32x4<Avx2>) -> i64x2<Avx2> {
4651 ({
4652 let y = _mm_xor_si128(a.into(), _mm_set1_epi32(i32::MIN));
4653 _mm_add_epi64(
4654 _mm_or_si128(y, _mm_set1_epi64x(-0x1_0000_0000)),
4655 _mm_srli_epi64::<32>(y),
4656 )
4657 })
4658 .simd_into(token)
4659 }
4660 );
4661 kernel(self, a)
4662 }
4663 #[inline(always)]
4664 fn pairwise_dot_product_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i64x2<Self> {
4665 let (low, high) = self.widening_mul_i32x4(a, b);
4666 let (even, odd) = self.deinterleave_i64x2(low, high);
4667 self.add_i64x2(even, odd)
4668 }
4669 #[inline(always)]
4670 fn narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
4671 crate::kernel!(
4672 #[inline(always)]
4673 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i16x8<Avx2> {
4674 let mask = _mm_set1_epi32(0xffff);
4675 _mm_packus_epi32(_mm_and_si128(a.into(), mask), _mm_and_si128(b.into(), mask))
4676 .simd_into(token)
4677 }
4678 );
4679 kernel(self, a, b)
4680 }
4681 #[inline(always)]
4682 fn saturating_narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
4683 crate::kernel!(
4684 #[inline(always)]
4685 fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i16x8<Avx2> {
4686 _mm_packs_epi32(a.into(), b.into()).simd_into(token)
4687 }
4688 );
4689 kernel(self, a, b)
4690 }
4691 #[inline(always)]
4692 fn relaxed_narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
4693 debug_assert!(
4694 a.as_slice()
4695 .iter()
4696 .chain(b.as_slice())
4697 .all(|&value| { value >= i16::MIN as i32 && value <= i16::MAX as i32 }),
4698 "relaxed_narrow inputs must fit in the destination type",
4699 );
4700 self.saturating_narrow_i32x4(a, b)
4701 }
4702 #[inline(always)]
4703 fn cvt_f32_i32x4(self, a: i32x4<Self>) -> f32x4<Self> {
4704 crate::kernel!(
4705 #[inline(always)]
4706 fn kernel(token: Avx2, a: i32x4<Avx2>) -> f32x4<Avx2> {
4707 _mm_cvtepi32_ps(a.into()).simd_into(token)
4708 }
4709 );
4710 kernel(self, a)
4711 }
4712 #[inline(always)]
4713 fn splat_u32x4(self, val: u32) -> u32x4<Self> {
4714 crate::kernel!(
4715 #[inline(always)]
4716 fn kernel(token: Avx2, val: u32) -> u32x4<Avx2> {
4717 _mm_set1_epi32(val.cast_signed()).simd_into(token)
4718 }
4719 );
4720 kernel(self, val)
4721 }
4722 #[inline(always)]
4723 fn reverse_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
4724 let indices: u8x16<Self> =
4725 [12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3].simd_into(self);
4726 self.swizzle_dyn_u32x4(a, indices)
4727 }
4728 #[inline(always)]
4729 fn slide_u32x4<const SHIFT: usize>(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4730 if SHIFT >= 4usize {
4731 return b;
4732 }
4733 let result = dyn_alignr_128(
4734 self,
4735 Bytes::to_bytes(b).val.0,
4736 Bytes::to_bytes(a).val.0,
4737 SHIFT * 4usize,
4738 );
4739 Bytes::from_bytes(u8x16 {
4740 val: crate::support::Aligned128(result),
4741 simd: self,
4742 })
4743 }
4744 #[inline(always)]
4745 fn compress_u32x4(self, values: u32x4<Self>, mask: mask32x4<Self>) -> u32x4<Self> {
4746 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
4747 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0);
4748 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
4749 }
4750 #[inline(always)]
4751 fn compress_merge_u32x4(
4752 self,
4753 values: u32x4<Self>,
4754 mask: mask32x4<Self>,
4755 merge: u32x4<Self>,
4756 ) -> u32x4<Self> {
4757 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
4758 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0);
4759 let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control);
4760 let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0));
4761 Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed))
4762 }
4763 #[inline(always)]
4764 fn expand_u32x4(self, values: u32x4<Self>, mask: mask32x4<Self>) -> u32x4<Self> {
4765 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
4766 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0);
4767 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
4768 }
4769 #[inline(always)]
4770 fn expand_merge_u32x4(
4771 self,
4772 values: u32x4<Self>,
4773 mask: mask32x4<Self>,
4774 merge: u32x4<Self>,
4775 ) -> u32x4<Self> {
4776 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
4777 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0);
4778 let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control));
4779 self.select_u32x4(mask, expanded, merge)
4780 }
4781 #[inline(always)]
4782 fn count_ones_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
4783 crate::kernel!(
4784 #[inline(always)]
4785 fn kernel(token: Avx2, a: u32x4<Avx2>) -> u32x4<Avx2> {
4786 let value = a.into();
4787 let nibble_mask = _mm_set1_epi8(0x0f);
4788 let lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4);
4789 let low = _mm_and_si128(value, nibble_mask);
4790 let high = _mm_and_si128(_mm_srli_epi16::<4>(value), nibble_mask);
4791 let byte_counts = _mm_add_epi8(
4792 _mm_shuffle_epi8(lookup, low),
4793 _mm_shuffle_epi8(lookup, high),
4794 );
4795 _mm_madd_epi16(
4796 _mm_maddubs_epi16(byte_counts, _mm_set1_epi8(1)),
4797 _mm_set1_epi16(1),
4798 )
4799 .simd_into(token)
4800 }
4801 );
4802 kernel(self, a)
4803 }
4804 #[inline(always)]
4805 fn count_zeros_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
4806 self.count_ones_u32x4(self.not_u32x4(a))
4807 }
4808 #[inline(always)]
4809 fn add_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4810 crate::kernel!(
4811 #[inline(always)]
4812 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
4813 _mm_add_epi32(a.into(), b.into()).simd_into(token)
4814 }
4815 );
4816 kernel(self, a, b)
4817 }
4818 #[inline(always)]
4819 fn saturating_add_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4820 crate::kernel!(
4821 #[inline(always)]
4822 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
4823 let a = a.into();
4824 let b = b.into();
4825 let threshold = _mm_xor_si128(b, _mm_set1_epi32(-1));
4826 _mm_add_epi32(_mm_min_epu32(a, threshold), b).simd_into(token)
4827 }
4828 );
4829 kernel(self, a, b)
4830 }
4831 #[inline(always)]
4832 fn sub_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4833 crate::kernel!(
4834 #[inline(always)]
4835 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
4836 _mm_sub_epi32(a.into(), b.into()).simd_into(token)
4837 }
4838 );
4839 kernel(self, a, b)
4840 }
4841 #[inline(always)]
4842 fn saturating_sub_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4843 crate::kernel!(
4844 #[inline(always)]
4845 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
4846 let a = a.into();
4847 let b = b.into();
4848 _mm_sub_epi32(_mm_max_epu32(a, b), b).simd_into(token)
4849 }
4850 );
4851 kernel(self, a, b)
4852 }
4853 #[inline(always)]
4854 fn mul_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4855 crate::kernel!(
4856 #[inline(always)]
4857 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
4858 _mm_mullo_epi32(a.into(), b.into()).simd_into(token)
4859 }
4860 );
4861 kernel(self, a, b)
4862 }
4863 #[inline(always)]
4864 fn and_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4865 crate::kernel!(
4866 #[inline(always)]
4867 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
4868 _mm_and_si128(a.into(), b.into()).simd_into(token)
4869 }
4870 );
4871 kernel(self, a, b)
4872 }
4873 #[inline(always)]
4874 fn or_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4875 crate::kernel!(
4876 #[inline(always)]
4877 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
4878 _mm_or_si128(a.into(), b.into()).simd_into(token)
4879 }
4880 );
4881 kernel(self, a, b)
4882 }
4883 #[inline(always)]
4884 fn xor_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4885 crate::kernel!(
4886 #[inline(always)]
4887 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
4888 _mm_xor_si128(a.into(), b.into()).simd_into(token)
4889 }
4890 );
4891 kernel(self, a, b)
4892 }
4893 #[inline(always)]
4894 fn not_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
4895 a ^ !0
4896 }
4897 #[inline(always)]
4898 fn shl_u32x4(self, a: u32x4<Self>, shift: u32) -> u32x4<Self> {
4899 crate::kernel!(
4900 #[inline(always)]
4901 fn kernel(token: Avx2, a: u32x4<Avx2>, shift: u32) -> u32x4<Avx2> {
4902 _mm_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
4903 }
4904 );
4905 kernel(self, a, shift)
4906 }
4907 #[inline(always)]
4908 fn shlv_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4909 crate::kernel!(
4910 #[inline(always)]
4911 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
4912 _mm_sllv_epi32(a.into(), b.into()).simd_into(token)
4913 }
4914 );
4915 kernel(self, a, b)
4916 }
4917 #[inline(always)]
4918 fn shr_u32x4(self, a: u32x4<Self>, shift: u32) -> u32x4<Self> {
4919 crate::kernel!(
4920 #[inline(always)]
4921 fn kernel(token: Avx2, a: u32x4<Avx2>, shift: u32) -> u32x4<Avx2> {
4922 _mm_srl_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
4923 }
4924 );
4925 kernel(self, a, shift)
4926 }
4927 #[inline(always)]
4928 fn shrv_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4929 crate::kernel!(
4930 #[inline(always)]
4931 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
4932 _mm_srlv_epi32(a.into(), b.into()).simd_into(token)
4933 }
4934 );
4935 kernel(self, a, b)
4936 }
4937 #[inline(always)]
4938 fn reduce_max_u32x4(self, a: u32x4<Self>) -> u32 {
4939 crate::kernel!(
4940 #[inline(always)]
4941 fn kernel(token: Avx2, a: u32x4<Avx2>) -> u32 {
4942 let reduced: __m128i = a.into();
4943 let shifted = _mm_srli_si128::<8>(reduced);
4944 let reduced = { _mm_max_epu32(reduced, shifted) };
4945 let shifted = _mm_srli_si128::<4>(reduced);
4946 let reduced = { _mm_max_epu32(reduced, shifted) };
4947 {
4948 let lanes: [u32; 4usize] = crate::transmute::checked_transmute_copy(&reduced);
4949 lanes[0]
4950 }
4951 }
4952 );
4953 kernel(self, a)
4954 }
4955 #[inline(always)]
4956 fn reduce_min_u32x4(self, a: u32x4<Self>) -> u32 {
4957 crate::kernel!(
4958 #[inline(always)]
4959 fn kernel(token: Avx2, a: u32x4<Avx2>) -> u32 {
4960 let reduced: __m128i = a.into();
4961 let shifted = _mm_srli_si128::<8>(reduced);
4962 let reduced = { _mm_min_epu32(reduced, shifted) };
4963 let shifted = _mm_srli_si128::<4>(reduced);
4964 let reduced = { _mm_min_epu32(reduced, shifted) };
4965 {
4966 let lanes: [u32; 4usize] = crate::transmute::checked_transmute_copy(&reduced);
4967 lanes[0]
4968 }
4969 }
4970 );
4971 kernel(self, a)
4972 }
4973 #[inline(always)]
4974 fn reduce_sum_u32x4(self, a: u32x4<Self>) -> u32 {
4975 crate::kernel!(
4976 #[inline(always)]
4977 fn kernel(token: Avx2, a: u32x4<Avx2>) -> u32 {
4978 let sum: __m128i = a.into();
4979 let sum = _mm_add_epi32(sum, _mm_srli_si128::<8>(sum));
4980 let sum = _mm_add_epi32(sum, _mm_srli_si128::<4>(sum));
4981 let lanes: [u32; 4usize] = crate::transmute::checked_transmute_copy(&sum);
4982 lanes[0]
4983 }
4984 );
4985 kernel(self, a)
4986 }
4987 #[inline(always)]
4988 fn reduce_product_u32x4(self, a: u32x4<Self>) -> u32 {
4989 let mul_level_0: [u32; 2usize] = [
4990 a[0usize].wrapping_mul(a[1usize]),
4991 a[2usize].wrapping_mul(a[3usize]),
4992 ];
4993 let mul_level_1: [u32; 1usize] = [mul_level_0[0usize].wrapping_mul(mul_level_0[1usize])];
4994 mul_level_1[0]
4995 }
4996 #[inline(always)]
4997 fn max_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4998 crate::kernel!(
4999 #[inline(always)]
5000 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
5001 _mm_max_epu32(a.into(), b.into()).simd_into(token)
5002 }
5003 );
5004 kernel(self, a, b)
5005 }
5006 #[inline(always)]
5007 fn min_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5008 crate::kernel!(
5009 #[inline(always)]
5010 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
5011 _mm_min_epu32(a.into(), b.into()).simd_into(token)
5012 }
5013 );
5014 kernel(self, a, b)
5015 }
5016 #[inline(always)]
5017 fn clamp_u32x4(self, a: u32x4<Self>, b: u32x4<Self>, c: u32x4<Self>) -> u32x4<Self> {
5018 debug_assert!(
5019 b.simd_le(c).all_true(),
5020 "clamp requires min <= max for all elements"
5021 );
5022 a.max(b).min(c)
5023 }
5024 #[inline(always)]
5025 fn simd_eq_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
5026 crate::kernel!(
5027 #[inline(always)]
5028 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> mask32x4<Avx2> {
5029 _mm_cmpeq_epi32(a.into(), b.into()).simd_into(token)
5030 }
5031 );
5032 kernel(self, a, b)
5033 }
5034 #[inline(always)]
5035 fn simd_lt_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
5036 crate::kernel!(
5037 #[inline(always)]
5038 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> mask32x4<Avx2> {
5039 {
5040 let sign_bit = _mm_set1_epi32(0x80000000u32.cast_signed());
5041 let lhs_signed = _mm_xor_si128(b.into(), sign_bit);
5042 let rhs_signed = _mm_xor_si128(a.into(), sign_bit);
5043 _mm_cmpgt_epi32(lhs_signed, rhs_signed)
5044 }
5045 .simd_into(token)
5046 }
5047 );
5048 kernel(self, a, b)
5049 }
5050 #[inline(always)]
5051 fn simd_le_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
5052 crate::kernel!(
5053 #[inline(always)]
5054 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> mask32x4<Avx2> {
5055 _mm_cmpeq_epi32(_mm_min_epu32(a.into(), b.into()), a.into()).simd_into(token)
5056 }
5057 );
5058 kernel(self, a, b)
5059 }
5060 #[inline(always)]
5061 fn zip_low_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5062 crate::kernel!(
5063 #[inline(always)]
5064 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
5065 _mm_unpacklo_epi32(a.into(), b.into()).simd_into(token)
5066 }
5067 );
5068 kernel(self, a, b)
5069 }
5070 #[inline(always)]
5071 fn zip_high_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5072 crate::kernel!(
5073 #[inline(always)]
5074 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
5075 _mm_unpackhi_epi32(a.into(), b.into()).simd_into(token)
5076 }
5077 );
5078 kernel(self, a, b)
5079 }
5080 #[inline(always)]
5081 fn unzip_low_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5082 crate::kernel!(
5083 #[inline(always)]
5084 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
5085 let t1 = _mm_shuffle_epi32::<0b11_01_10_00>(a.into());
5086 let t2 = _mm_shuffle_epi32::<0b11_01_10_00>(b.into());
5087 _mm_unpacklo_epi64(t1, t2).simd_into(token)
5088 }
5089 );
5090 kernel(self, a, b)
5091 }
5092 #[inline(always)]
5093 fn unzip_high_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5094 crate::kernel!(
5095 #[inline(always)]
5096 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
5097 let t1 = _mm_shuffle_epi32::<0b11_01_10_00>(a.into());
5098 let t2 = _mm_shuffle_epi32::<0b11_01_10_00>(b.into());
5099 _mm_unpackhi_epi64(t1, t2).simd_into(token)
5100 }
5101 );
5102 kernel(self, a, b)
5103 }
5104 #[inline(always)]
5105 fn interleave_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u32x4<Self>, u32x4<Self>) {
5106 (self.zip_low_u32x4(a, b), self.zip_high_u32x4(a, b))
5107 }
5108 #[inline(always)]
5109 fn deinterleave_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u32x4<Self>, u32x4<Self>) {
5110 (self.unzip_low_u32x4(a, b), self.unzip_high_u32x4(a, b))
5111 }
5112 #[inline(always)]
5113 fn select_u32x4(self, a: mask32x4<Self>, b: u32x4<Self>, c: u32x4<Self>) -> u32x4<Self> {
5114 crate::kernel!(
5115 #[inline(always)]
5116 fn kernel(
5117 token: Avx2,
5118 a: mask32x4<Avx2>,
5119 b: u32x4<Avx2>,
5120 c: u32x4<Avx2>,
5121 ) -> u32x4<Avx2> {
5122 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
5123 }
5124 );
5125 kernel(self, a, b, c)
5126 }
5127 #[inline(always)]
5128 fn combine_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x8<Self> {
5129 crate::kernel!(
5130 #[inline(always)]
5131 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x8<Avx2> {
5132 _mm256_setr_m128i(a.into(), b.into()).simd_into(token)
5133 }
5134 );
5135 kernel(self, a, b)
5136 }
5137 #[inline(always)]
5138 fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4<Self>; 4usize] {
5139 crate::kernel!(
5140 #[inline(always)]
5141 fn kernel(token: Avx2, src: &[u32; 16usize]) -> [u32x4<Avx2>; 4usize] {
5142 let (chunks, []) = src.as_chunks::<4usize>() else {
5143 unreachable!()
5144 };
5145 let v0: __m128i =
5146 crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]);
5147 let v1: __m128i =
5148 crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]);
5149 let v2: __m128i =
5150 crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]);
5151 let v3: __m128i =
5152 crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]);
5153 let tmp0 = _mm_unpacklo_epi32(v0, v1);
5154 let tmp1 = _mm_unpackhi_epi32(v0, v1);
5155 let tmp2 = _mm_unpacklo_epi32(v2, v3);
5156 let tmp3 = _mm_unpackhi_epi32(v2, v3);
5157 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
5158 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
5159 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
5160 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
5161 [
5162 out0.simd_into(token),
5163 out1.simd_into(token),
5164 out2.simd_into(token),
5165 out3.simd_into(token),
5166 ]
5167 }
5168 );
5169 kernel(self, src)
5170 }
5171 #[inline(always)]
5172 fn store_four_interleaved_u32x4(
5173 self,
5174 vectors: [u32x4<Self>; 4usize],
5175 dest: &mut [u32; 16usize],
5176 ) -> () {
5177 crate::kernel!(
5178 #[inline(always)]
5179 fn kernel(
5180 token: Avx2,
5181 vectors: [u32x4<Avx2>; 4usize],
5182 dest: &mut [u32; 16usize],
5183 ) -> () {
5184 let _ = token;
5185 let v0: __m128i = vectors[0].into();
5186 let v1: __m128i = vectors[1].into();
5187 let v2: __m128i = vectors[2].into();
5188 let v3: __m128i = vectors[3].into();
5189 let tmp0 = _mm_unpacklo_epi32(v0, v1);
5190 let tmp1 = _mm_unpackhi_epi32(v0, v1);
5191 let tmp2 = _mm_unpacklo_epi32(v2, v3);
5192 let tmp3 = _mm_unpackhi_epi32(v2, v3);
5193 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
5194 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
5195 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
5196 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
5197 let (chunks, []) = dest.as_chunks_mut::<4usize>() else {
5198 unreachable!()
5199 };
5200 crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>(
5201 out0,
5202 &mut chunks[0],
5203 );
5204 crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>(
5205 out1,
5206 &mut chunks[1],
5207 );
5208 crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>(
5209 out2,
5210 &mut chunks[2],
5211 );
5212 crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>(
5213 out3,
5214 &mut chunks[3],
5215 );
5216 }
5217 );
5218 kernel(self, vectors, dest);
5219 }
5220 #[inline(always)]
5221 fn widen_u32x4(self, a: u32x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
5222 crate::kernel!(
5223 #[inline(always)]
5224 fn kernel(token: Avx2, a: u32x4<Avx2>) -> (u64x2<Avx2>, u64x2<Avx2>) {
5225 let raw = a.into();
5226 (
5227 _mm_cvtepu32_epi64(raw).simd_into(token),
5228 _mm_cvtepu32_epi64(_mm_srli_si128::<8>(raw)).simd_into(token),
5229 )
5230 }
5231 );
5232 kernel(self, a)
5233 }
5234 #[inline(always)]
5235 fn widening_mul_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
5236 let (a0, a1) = self.widen_u32x4(a);
5237 let (b0, b1) = self.widen_u32x4(b);
5238 (self.mul_u64x2(a0, b0), self.mul_u64x2(a1, b1))
5239 }
5240 #[inline(always)]
5241 fn widening_add_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
5242 let (a0, a1) = self.widen_u32x4(a);
5243 let (b0, b1) = self.widen_u32x4(b);
5244 (self.add_u64x2(a0, b0), self.add_u64x2(a1, b1))
5245 }
5246 #[inline(always)]
5247 fn pairwise_widening_add_u32x4(self, a: u32x4<Self>) -> u64x2<Self> {
5248 crate::kernel!(
5249 #[inline(always)]
5250 fn kernel(token: Avx2, a: u32x4<Avx2>) -> u64x2<Avx2> {
5251 ({
5252 let a = a.into();
5253 _mm_add_epi64(
5254 _mm_and_si128(a, _mm_set1_epi64x(4294967295)),
5255 _mm_srli_epi64::<32>(a),
5256 )
5257 })
5258 .simd_into(token)
5259 }
5260 );
5261 kernel(self, a)
5262 }
5263 #[inline(always)]
5264 fn pairwise_dot_product_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u64x2<Self> {
5265 let (low, high) = self.widening_mul_u32x4(a, b);
5266 let (even, odd) = self.deinterleave_u64x2(low, high);
5267 self.add_u64x2(even, odd)
5268 }
5269 #[inline(always)]
5270 fn narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
5271 crate::kernel!(
5272 #[inline(always)]
5273 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u16x8<Avx2> {
5274 let mask = _mm_set1_epi32(0xffff);
5275 _mm_packus_epi32(_mm_and_si128(a.into(), mask), _mm_and_si128(b.into(), mask))
5276 .simd_into(token)
5277 }
5278 );
5279 kernel(self, a, b)
5280 }
5281 #[inline(always)]
5282 fn saturating_narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
5283 crate::kernel!(
5284 #[inline(always)]
5285 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u16x8<Avx2> {
5286 {
5287 let max = _mm_set1_epi32(i32::MAX);
5288 _mm_packus_epi32(_mm_min_epu32(a.into(), max), _mm_min_epu32(b.into(), max))
5289 }
5290 .simd_into(token)
5291 }
5292 );
5293 kernel(self, a, b)
5294 }
5295 #[inline(always)]
5296 fn relaxed_narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
5297 crate::kernel!(
5298 #[inline(always)]
5299 fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u16x8<Avx2> {
5300 debug_assert!(
5301 a.as_slice()
5302 .iter()
5303 .chain(b.as_slice())
5304 .all(|&value| { value >= u16::MIN as u32 && value <= u16::MAX as u32 }),
5305 "relaxed_narrow inputs must fit in the destination type",
5306 );
5307 _mm_packus_epi32(a.into(), b.into()).simd_into(token)
5308 }
5309 );
5310 kernel(self, a, b)
5311 }
5312 #[inline(always)]
5313 fn cvt_f32_u32x4(self, a: u32x4<Self>) -> f32x4<Self> {
5314 crate::kernel!(
5315 #[inline(always)]
5316 fn kernel(token: Avx2, a: u32x4<Avx2>) -> f32x4<Avx2> {
5317 let a = a.into();
5318 let lo = _mm_blend_epi16::<0xAA>(a, _mm_set1_epi32(0x4B000000));
5319 let hi =
5320 _mm_blend_epi16::<0xAA>(_mm_srli_epi32::<16>(a), _mm_set1_epi32(0x53000000));
5321 let fhi = _mm_sub_ps(
5322 _mm_castsi128_ps(hi),
5323 _mm_set1_ps(f32::from_bits(0x53000080)),
5324 );
5325 let result = _mm_add_ps(_mm_castsi128_ps(lo), fhi);
5326 result.simd_into(token)
5327 }
5328 );
5329 kernel(self, a)
5330 }
5331 #[inline(always)]
5332 fn splat_mask32x4(self, val: bool) -> mask32x4<Self> {
5333 crate::kernel!(
5334 #[inline(always)]
5335 fn kernel(token: Avx2, val: bool) -> mask32x4<Avx2> {
5336 let val: i32 = if val { !0 } else { 0 };
5337 _mm_set1_epi32(val).simd_into(token)
5338 }
5339 );
5340 kernel(self, val)
5341 }
5342 #[inline(always)]
5343 fn from_bitmask_mask32x4(self, bits: u64) -> mask32x4<Self> {
5344 crate::kernel!(
5345 #[inline(always)]
5346 fn kernel(token: Avx2, bits: u64) -> mask32x4<Avx2> {
5347 {
5348 let bit_lanes = _mm_set1_epi32(bits as i32);
5349 let bit_mask = _mm_setr_epi32(1, 2, 4, 8);
5350 _mm_cmpeq_epi32(_mm_and_si128(bit_lanes, bit_mask), bit_mask)
5351 }
5352 .simd_into(token)
5353 }
5354 );
5355 kernel(self, bits)
5356 }
5357 #[inline(always)]
5358 fn to_bitmask_mask32x4(self, a: mask32x4<Self>) -> u64 {
5359 crate::kernel!(
5360 #[inline(always)]
5361 fn kernel(token: Avx2, a: mask32x4<Avx2>) -> u64 {
5362 _mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 as u64
5363 }
5364 );
5365 kernel(self, a)
5366 }
5367 #[inline(always)]
5368 fn from_vector_mask32x4(self, vector: i32x4<Self>) -> mask32x4<Self> {
5369 mask32x4 {
5370 val: vector.val,
5371 simd: self,
5372 }
5373 }
5374 #[inline(always)]
5375 fn to_vector_mask32x4(self, a: mask32x4<Self>) -> i32x4<Self> {
5376 i32x4 {
5377 val: a.val,
5378 simd: self,
5379 }
5380 }
5381 #[inline(always)]
5382 fn set_mask32x4(self, a: &mut mask32x4<Self>, index: usize, value: bool) -> () {
5383 assert!(
5384 index < 4usize,
5385 "mask lane index {index} is out of bounds for {} lanes",
5386 4usize
5387 );
5388 let mut lanes: [i32; 4usize] = (*a).into();
5389 lanes[index] = if value { !0 } else { 0 };
5390 *a = lanes.simd_into(self);
5391 }
5392 #[inline(always)]
5393 fn rotate_elements_left_mask32x4<const OFFSET: usize>(
5394 self,
5395 a: mask32x4<Self>,
5396 ) -> mask32x4<Self> {
5397 let int = i32x4 {
5398 val: crate::transmute::checked_transmute_copy(&a.val),
5399 simd: self,
5400 };
5401 let rotated = int.rotate_elements_left::<OFFSET>();
5402 mask32x4 {
5403 val: crate::transmute::checked_transmute_copy(&rotated.val),
5404 simd: self,
5405 }
5406 }
5407 #[inline(always)]
5408 fn rotate_elements_right_mask32x4<const OFFSET: usize>(
5409 self,
5410 a: mask32x4<Self>,
5411 ) -> mask32x4<Self> {
5412 let int = i32x4 {
5413 val: crate::transmute::checked_transmute_copy(&a.val),
5414 simd: self,
5415 };
5416 let rotated = int.rotate_elements_right::<OFFSET>();
5417 mask32x4 {
5418 val: crate::transmute::checked_transmute_copy(&rotated.val),
5419 simd: self,
5420 }
5421 }
5422 #[inline(always)]
5423 fn and_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
5424 crate::kernel!(
5425 #[inline(always)]
5426 fn kernel(token: Avx2, a: mask32x4<Avx2>, b: mask32x4<Avx2>) -> mask32x4<Avx2> {
5427 _mm_and_si128(a.into(), b.into()).simd_into(token)
5428 }
5429 );
5430 kernel(self, a, b)
5431 }
5432 #[inline(always)]
5433 fn or_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
5434 crate::kernel!(
5435 #[inline(always)]
5436 fn kernel(token: Avx2, a: mask32x4<Avx2>, b: mask32x4<Avx2>) -> mask32x4<Avx2> {
5437 _mm_or_si128(a.into(), b.into()).simd_into(token)
5438 }
5439 );
5440 kernel(self, a, b)
5441 }
5442 #[inline(always)]
5443 fn xor_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
5444 crate::kernel!(
5445 #[inline(always)]
5446 fn kernel(token: Avx2, a: mask32x4<Avx2>, b: mask32x4<Avx2>) -> mask32x4<Avx2> {
5447 _mm_xor_si128(a.into(), b.into()).simd_into(token)
5448 }
5449 );
5450 kernel(self, a, b)
5451 }
5452 #[inline(always)]
5453 fn not_mask32x4(self, a: mask32x4<Self>) -> mask32x4<Self> {
5454 self.xor_mask32x4(a, self.splat_mask32x4(true))
5455 }
5456 #[inline(always)]
5457 fn reverse_mask32x4(self, a: mask32x4<Self>) -> mask32x4<Self> {
5458 let lanes = i32x4 {
5459 val: crate::transmute::checked_transmute_copy(&a.val),
5460 simd: self,
5461 };
5462 let reversed = self.reverse_i32x4(lanes);
5463 mask32x4 {
5464 val: crate::transmute::checked_transmute_copy(&reversed.val),
5465 simd: self,
5466 }
5467 }
5468 #[inline(always)]
5469 fn select_mask32x4(
5470 self,
5471 a: mask32x4<Self>,
5472 b: mask32x4<Self>,
5473 c: mask32x4<Self>,
5474 ) -> mask32x4<Self> {
5475 crate::kernel!(
5476 #[inline(always)]
5477 fn kernel(
5478 token: Avx2,
5479 a: mask32x4<Avx2>,
5480 b: mask32x4<Avx2>,
5481 c: mask32x4<Avx2>,
5482 ) -> mask32x4<Avx2> {
5483 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
5484 }
5485 );
5486 kernel(self, a, b, c)
5487 }
5488 #[inline(always)]
5489 fn simd_eq_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
5490 crate::kernel!(
5491 #[inline(always)]
5492 fn kernel(token: Avx2, a: mask32x4<Avx2>, b: mask32x4<Avx2>) -> mask32x4<Avx2> {
5493 _mm_cmpeq_epi32(a.into(), b.into()).simd_into(token)
5494 }
5495 );
5496 kernel(self, a, b)
5497 }
5498 #[inline(always)]
5499 fn any_true_mask32x4(self, a: mask32x4<Self>) -> bool {
5500 crate::kernel!(
5501 #[inline(always)]
5502 fn kernel(token: Avx2, a: mask32x4<Avx2>) -> bool {
5503 _mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 != 0
5504 }
5505 );
5506 kernel(self, a)
5507 }
5508 #[inline(always)]
5509 fn all_true_mask32x4(self, a: mask32x4<Self>) -> bool {
5510 crate::kernel!(
5511 #[inline(always)]
5512 fn kernel(token: Avx2, a: mask32x4<Avx2>) -> bool {
5513 _mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 == 0b1111
5514 }
5515 );
5516 kernel(self, a)
5517 }
5518 #[inline(always)]
5519 fn any_false_mask32x4(self, a: mask32x4<Self>) -> bool {
5520 crate::kernel!(
5521 #[inline(always)]
5522 fn kernel(token: Avx2, a: mask32x4<Avx2>) -> bool {
5523 _mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 != 0b1111
5524 }
5525 );
5526 kernel(self, a)
5527 }
5528 #[inline(always)]
5529 fn all_false_mask32x4(self, a: mask32x4<Self>) -> bool {
5530 crate::kernel!(
5531 #[inline(always)]
5532 fn kernel(token: Avx2, a: mask32x4<Avx2>) -> bool {
5533 _mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 == 0
5534 }
5535 );
5536 kernel(self, a)
5537 }
5538 #[inline(always)]
5539 fn combine_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x8<Self> {
5540 crate::kernel!(
5541 #[inline(always)]
5542 fn kernel(token: Avx2, a: mask32x4<Avx2>, b: mask32x4<Avx2>) -> mask32x8<Avx2> {
5543 _mm256_setr_m128i(a.into(), b.into()).simd_into(token)
5544 }
5545 );
5546 kernel(self, a, b)
5547 }
5548 #[inline(always)]
5549 fn widen_mask32x4(self, a: mask32x4<Self>) -> (mask64x2<Self>, mask64x2<Self>) {
5550 crate::kernel!(
5551 #[inline(always)]
5552 fn kernel(token: Avx2, a: mask32x4<Avx2>) -> (mask64x2<Avx2>, mask64x2<Avx2>) {
5553 let raw = a.into();
5554 (
5555 _mm_unpacklo_epi32(raw, raw).simd_into(token),
5556 _mm_unpackhi_epi32(raw, raw).simd_into(token),
5557 )
5558 }
5559 );
5560 kernel(self, a)
5561 }
5562 #[inline(always)]
5563 fn narrow_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask16x8<Self> {
5564 crate::kernel!(
5565 #[inline(always)]
5566 fn kernel(token: Avx2, a: mask32x4<Avx2>, b: mask32x4<Avx2>) -> mask16x8<Avx2> {
5567 _mm_packs_epi32(a.into(), b.into()).simd_into(token)
5568 }
5569 );
5570 kernel(self, a, b)
5571 }
5572 #[inline(always)]
5573 fn abs_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5574 crate::kernel!(
5575 #[inline(always)]
5576 fn kernel(token: Avx2, a: f64x2<Avx2>) -> f64x2<Avx2> {
5577 _mm_andnot_pd(_mm_set1_pd(-0.0), a.into()).simd_into(token)
5578 }
5579 );
5580 kernel(self, a)
5581 }
5582 #[inline(always)]
5583 fn splat_f64x2(self, val: f64) -> f64x2<Self> {
5584 crate::kernel!(
5585 #[inline(always)]
5586 fn kernel(token: Avx2, val: f64) -> f64x2<Avx2> {
5587 _mm_set1_pd(val).simd_into(token)
5588 }
5589 );
5590 kernel(self, val)
5591 }
5592 #[inline(always)]
5593 fn reverse_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5594 let indices: u8x16<Self> =
5595 [8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7].simd_into(self);
5596 self.swizzle_dyn_f64x2(a, indices)
5597 }
5598 #[inline(always)]
5599 fn slide_f64x2<const SHIFT: usize>(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5600 if SHIFT >= 2usize {
5601 return b;
5602 }
5603 let result = dyn_alignr_128(
5604 self,
5605 Bytes::to_bytes(b).val.0,
5606 Bytes::to_bytes(a).val.0,
5607 SHIFT * 8usize,
5608 );
5609 Bytes::from_bytes(u8x16 {
5610 val: crate::support::Aligned128(result),
5611 simd: self,
5612 })
5613 }
5614 #[inline(always)]
5615 fn compress_f64x2(self, values: f64x2<Self>, mask: mask64x2<Self>) -> f64x2<Self> {
5616 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
5617 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0);
5618 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
5619 }
5620 #[inline(always)]
5621 fn compress_merge_f64x2(
5622 self,
5623 values: f64x2<Self>,
5624 mask: mask64x2<Self>,
5625 merge: f64x2<Self>,
5626 ) -> f64x2<Self> {
5627 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
5628 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0);
5629 let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control);
5630 let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0));
5631 Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed))
5632 }
5633 #[inline(always)]
5634 fn expand_f64x2(self, values: f64x2<Self>, mask: mask64x2<Self>) -> f64x2<Self> {
5635 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
5636 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0);
5637 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
5638 }
5639 #[inline(always)]
5640 fn expand_merge_f64x2(
5641 self,
5642 values: f64x2<Self>,
5643 mask: mask64x2<Self>,
5644 merge: f64x2<Self>,
5645 ) -> f64x2<Self> {
5646 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
5647 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0);
5648 let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control));
5649 self.select_f64x2(mask, expanded, merge)
5650 }
5651 #[inline(always)]
5652 fn neg_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5653 crate::kernel!(
5654 #[inline(always)]
5655 fn kernel(token: Avx2, a: f64x2<Avx2>) -> f64x2<Avx2> {
5656 _mm_xor_pd(a.into(), _mm_set1_pd(-0.0)).simd_into(token)
5657 }
5658 );
5659 kernel(self, a)
5660 }
5661 #[inline(always)]
5662 fn sqrt_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5663 crate::kernel!(
5664 #[inline(always)]
5665 fn kernel(token: Avx2, a: f64x2<Avx2>) -> f64x2<Avx2> {
5666 _mm_sqrt_pd(a.into()).simd_into(token)
5667 }
5668 );
5669 kernel(self, a)
5670 }
5671 #[inline(always)]
5672 fn approximate_recip_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5673 1.0 / a
5674 }
5675 #[inline(always)]
5676 fn add_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5677 crate::kernel!(
5678 #[inline(always)]
5679 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
5680 _mm_add_pd(a.into(), b.into()).simd_into(token)
5681 }
5682 );
5683 kernel(self, a, b)
5684 }
5685 #[inline(always)]
5686 fn sub_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5687 crate::kernel!(
5688 #[inline(always)]
5689 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
5690 _mm_sub_pd(a.into(), b.into()).simd_into(token)
5691 }
5692 );
5693 kernel(self, a, b)
5694 }
5695 #[inline(always)]
5696 fn mul_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5697 crate::kernel!(
5698 #[inline(always)]
5699 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
5700 _mm_mul_pd(a.into(), b.into()).simd_into(token)
5701 }
5702 );
5703 kernel(self, a, b)
5704 }
5705 #[inline(always)]
5706 fn div_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5707 crate::kernel!(
5708 #[inline(always)]
5709 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
5710 _mm_div_pd(a.into(), b.into()).simd_into(token)
5711 }
5712 );
5713 kernel(self, a, b)
5714 }
5715 #[inline(always)]
5716 fn copysign_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5717 crate::kernel!(
5718 #[inline(always)]
5719 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
5720 let mask = _mm_set1_pd(-0.0);
5721 _mm_or_pd(_mm_and_pd(mask, b.into()), _mm_andnot_pd(mask, a.into()))
5722 .simd_into(token)
5723 }
5724 );
5725 kernel(self, a, b)
5726 }
5727 #[inline(always)]
5728 fn reduce_max_f64x2(self, a: f64x2<Self>) -> f64 {
5729 crate::kernel!(
5730 #[inline(always)]
5731 fn kernel(token: Avx2, a: f64x2<Avx2>) -> f64 {
5732 let reduced: __m128d = a.into();
5733 let shifted = _mm_castsi128_pd(_mm_srli_si128::<8>(_mm_castpd_si128(reduced)));
5734 let reduced = { _mm_max_pd(reduced, shifted) };
5735 _mm_cvtsd_f64(reduced)
5736 }
5737 );
5738 kernel(self, a)
5739 }
5740 #[inline(always)]
5741 fn reduce_min_f64x2(self, a: f64x2<Self>) -> f64 {
5742 crate::kernel!(
5743 #[inline(always)]
5744 fn kernel(token: Avx2, a: f64x2<Avx2>) -> f64 {
5745 let reduced: __m128d = a.into();
5746 let shifted = _mm_castsi128_pd(_mm_srli_si128::<8>(_mm_castpd_si128(reduced)));
5747 let reduced = { _mm_min_pd(reduced, shifted) };
5748 _mm_cvtsd_f64(reduced)
5749 }
5750 );
5751 kernel(self, a)
5752 }
5753 #[inline(always)]
5754 fn reduce_max_precise_f64x2(self, a: f64x2<Self>) -> f64 {
5755 crate::kernel!(
5756 #[inline(always)]
5757 fn kernel(token: Avx2, a: f64x2<Avx2>) -> f64 {
5758 let reduced: __m128d = a.into();
5759 let shifted = _mm_castsi128_pd(_mm_srli_si128::<8>(_mm_castpd_si128(reduced)));
5760 let reduced = {
5761 let intermediate = _mm_max_pd(reduced, shifted);
5762 let b_is_nan = _mm_cmpunord_pd(shifted, shifted);
5763 _mm_blendv_pd(intermediate, reduced, b_is_nan)
5764 };
5765 _mm_cvtsd_f64(reduced)
5766 }
5767 );
5768 kernel(self, a)
5769 }
5770 #[inline(always)]
5771 fn reduce_min_precise_f64x2(self, a: f64x2<Self>) -> f64 {
5772 crate::kernel!(
5773 #[inline(always)]
5774 fn kernel(token: Avx2, a: f64x2<Avx2>) -> f64 {
5775 let reduced: __m128d = a.into();
5776 let shifted = _mm_castsi128_pd(_mm_srli_si128::<8>(_mm_castpd_si128(reduced)));
5777 let reduced = {
5778 let intermediate = _mm_min_pd(reduced, shifted);
5779 let b_is_nan = _mm_cmpunord_pd(shifted, shifted);
5780 _mm_blendv_pd(intermediate, reduced, b_is_nan)
5781 };
5782 _mm_cvtsd_f64(reduced)
5783 }
5784 );
5785 kernel(self, a)
5786 }
5787 #[inline(always)]
5788 fn reduce_sum_f64x2(self, a: f64x2<Self>) -> f64 {
5789 crate::kernel!(
5790 #[inline(always)]
5791 fn kernel(token: Avx2, a: f64x2<Avx2>) -> f64 {
5792 let a: __m128d = a.into();
5793 _mm_cvtsd_f64(_mm_add_sd(a, _mm_unpackhi_pd(a, a)))
5794 }
5795 );
5796 kernel(self, a)
5797 }
5798 #[inline(always)]
5799 fn reduce_product_f64x2(self, a: f64x2<Self>) -> f64 {
5800 crate::kernel!(
5801 #[inline(always)]
5802 fn kernel(token: Avx2, a: f64x2<Avx2>) -> f64 {
5803 let a: __m128d = a.into();
5804 _mm_cvtsd_f64(_mm_mul_sd(a, _mm_unpackhi_pd(a, a)))
5805 }
5806 );
5807 kernel(self, a)
5808 }
5809 #[inline(always)]
5810 fn max_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5811 crate::kernel!(
5812 #[inline(always)]
5813 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
5814 _mm_max_pd(a.into(), b.into()).simd_into(token)
5815 }
5816 );
5817 kernel(self, a, b)
5818 }
5819 #[inline(always)]
5820 fn min_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5821 crate::kernel!(
5822 #[inline(always)]
5823 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
5824 _mm_min_pd(a.into(), b.into()).simd_into(token)
5825 }
5826 );
5827 kernel(self, a, b)
5828 }
5829 #[inline(always)]
5830 fn max_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5831 crate::kernel!(
5832 #[inline(always)]
5833 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
5834 let intermediate = _mm_max_pd(a.into(), b.into());
5835 let b_is_nan = _mm_cmpunord_pd(b.into(), b.into());
5836 _mm_blendv_pd(intermediate, a.into(), b_is_nan).simd_into(token)
5837 }
5838 );
5839 kernel(self, a, b)
5840 }
5841 #[inline(always)]
5842 fn min_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5843 crate::kernel!(
5844 #[inline(always)]
5845 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
5846 let intermediate = _mm_min_pd(a.into(), b.into());
5847 let b_is_nan = _mm_cmpunord_pd(b.into(), b.into());
5848 _mm_blendv_pd(intermediate, a.into(), b_is_nan).simd_into(token)
5849 }
5850 );
5851 kernel(self, a, b)
5852 }
5853 #[inline(always)]
5854 fn clamp_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
5855 crate::kernel!(
5856 #[inline(always)]
5857 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>, c: f64x2<Avx2>) -> f64x2<Avx2> {
5858 debug_assert!(
5859 b.simd_le(c).all_true(),
5860 "clamp requires min <= max for all elements"
5861 );
5862 _mm_min_pd(c.into(), _mm_max_pd(b.into(), a.into())).simd_into(token)
5863 }
5864 );
5865 kernel(self, a, b, c)
5866 }
5867 #[inline(always)]
5868 fn simd_eq_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
5869 crate::kernel!(
5870 #[inline(always)]
5871 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> mask64x2<Avx2> {
5872 _mm_castpd_si128(_mm_cmpeq_pd(a.into(), b.into())).simd_into(token)
5873 }
5874 );
5875 kernel(self, a, b)
5876 }
5877 #[inline(always)]
5878 fn simd_ne_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
5879 crate::kernel!(
5880 #[inline(always)]
5881 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> mask64x2<Avx2> {
5882 _mm_castpd_si128(_mm_cmpneq_pd(a.into(), b.into())).simd_into(token)
5883 }
5884 );
5885 kernel(self, a, b)
5886 }
5887 #[inline(always)]
5888 fn simd_lt_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
5889 crate::kernel!(
5890 #[inline(always)]
5891 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> mask64x2<Avx2> {
5892 _mm_castpd_si128(_mm_cmplt_pd(a.into(), b.into())).simd_into(token)
5893 }
5894 );
5895 kernel(self, a, b)
5896 }
5897 #[inline(always)]
5898 fn simd_le_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
5899 crate::kernel!(
5900 #[inline(always)]
5901 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> mask64x2<Avx2> {
5902 _mm_castpd_si128(_mm_cmple_pd(a.into(), b.into())).simd_into(token)
5903 }
5904 );
5905 kernel(self, a, b)
5906 }
5907 #[inline(always)]
5908 fn zip_low_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5909 crate::kernel!(
5910 #[inline(always)]
5911 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
5912 _mm_unpacklo_pd(a.into(), b.into()).simd_into(token)
5913 }
5914 );
5915 kernel(self, a, b)
5916 }
5917 #[inline(always)]
5918 fn zip_high_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5919 crate::kernel!(
5920 #[inline(always)]
5921 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
5922 _mm_unpackhi_pd(a.into(), b.into()).simd_into(token)
5923 }
5924 );
5925 kernel(self, a, b)
5926 }
5927 #[inline(always)]
5928 fn unzip_low_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5929 crate::kernel!(
5930 #[inline(always)]
5931 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
5932 _mm_shuffle_pd::<0b00>(a.into(), b.into()).simd_into(token)
5933 }
5934 );
5935 kernel(self, a, b)
5936 }
5937 #[inline(always)]
5938 fn unzip_high_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5939 crate::kernel!(
5940 #[inline(always)]
5941 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
5942 _mm_shuffle_pd::<0b11>(a.into(), b.into()).simd_into(token)
5943 }
5944 );
5945 kernel(self, a, b)
5946 }
5947 #[inline(always)]
5948 fn interleave_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> (f64x2<Self>, f64x2<Self>) {
5949 (self.zip_low_f64x2(a, b), self.zip_high_f64x2(a, b))
5950 }
5951 #[inline(always)]
5952 fn deinterleave_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> (f64x2<Self>, f64x2<Self>) {
5953 (self.unzip_low_f64x2(a, b), self.unzip_high_f64x2(a, b))
5954 }
5955 #[inline(always)]
5956 fn mul_add_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
5957 crate::kernel!(
5958 #[inline(always)]
5959 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>, c: f64x2<Avx2>) -> f64x2<Avx2> {
5960 _mm_fmadd_pd(a.into(), b.into(), c.into()).simd_into(token)
5961 }
5962 );
5963 kernel(self, a, b, c)
5964 }
5965 #[inline(always)]
5966 fn mul_add_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
5967 self.mul_add_f64x2(a, b, c)
5968 }
5969 #[inline(always)]
5970 fn mul_sub_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
5971 crate::kernel!(
5972 #[inline(always)]
5973 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>, c: f64x2<Avx2>) -> f64x2<Avx2> {
5974 _mm_fmsub_pd(a.into(), b.into(), c.into()).simd_into(token)
5975 }
5976 );
5977 kernel(self, a, b, c)
5978 }
5979 #[inline(always)]
5980 fn mul_sub_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
5981 self.mul_sub_f64x2(a, b, c)
5982 }
5983 #[inline(always)]
5984 fn floor_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5985 crate::kernel!(
5986 #[inline(always)]
5987 fn kernel(token: Avx2, a: f64x2<Avx2>) -> f64x2<Avx2> {
5988 _mm_round_pd::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
5989 .simd_into(token)
5990 }
5991 );
5992 kernel(self, a)
5993 }
5994 #[inline(always)]
5995 fn ceil_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5996 crate::kernel!(
5997 #[inline(always)]
5998 fn kernel(token: Avx2, a: f64x2<Avx2>) -> f64x2<Avx2> {
5999 _mm_round_pd::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
6000 .simd_into(token)
6001 }
6002 );
6003 kernel(self, a)
6004 }
6005 #[inline(always)]
6006 fn round_ties_even_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6007 crate::kernel!(
6008 #[inline(always)]
6009 fn kernel(token: Avx2, a: f64x2<Avx2>) -> f64x2<Avx2> {
6010 _mm_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
6011 .simd_into(token)
6012 }
6013 );
6014 kernel(self, a)
6015 }
6016 #[inline(always)]
6017 fn round_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6018 let bias = self.splat_f64x2(const { f64::next_down(0.5) });
6019 (a + self.copysign_f64x2(bias, a)).trunc()
6020 }
6021 #[inline(always)]
6022 fn fract_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6023 a - self.trunc_f64x2(a)
6024 }
6025 #[inline(always)]
6026 fn trunc_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6027 crate::kernel!(
6028 #[inline(always)]
6029 fn kernel(token: Avx2, a: f64x2<Avx2>) -> f64x2<Avx2> {
6030 _mm_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
6031 .simd_into(token)
6032 }
6033 );
6034 kernel(self, a)
6035 }
6036 #[inline(always)]
6037 fn to_degrees_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6038 a * const { f64::to_degrees(1.0) }
6039 }
6040 #[inline(always)]
6041 fn to_radians_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6042 a * const { f64::to_radians(1.0) }
6043 }
6044 #[inline(always)]
6045 fn is_nan_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6046 a.simd_ne(a)
6047 }
6048 #[inline(always)]
6049 fn is_infinite_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6050 a.abs().simd_eq(f64::INFINITY)
6051 }
6052 #[inline(always)]
6053 fn is_finite_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6054 a.abs().simd_lt(f64::INFINITY)
6055 }
6056 #[inline(always)]
6057 fn is_subnormal_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6058 let i: i64x2<Self> = a.bitcast();
6059 (i & 0x7FF0_0000_0000_0000_i64).simd_eq(0) & (i & 0x000F_FFFF_FFFF_FFFF_i64).simd_ne(0)
6060 }
6061 #[inline(always)]
6062 fn is_normal_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6063 let i: i64x2<Self> = a.bitcast();
6064 let exp = i & 0x7FF0_0000_0000_0000_i64;
6065 exp.simd_ne(0) & exp.simd_ne(0x7FF0_0000_0000_0000_i64)
6066 }
6067 #[inline(always)]
6068 fn is_sign_positive_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6069 let i: i64x2<Self> = a.bitcast();
6070 (i & i64::MIN).simd_eq(0)
6071 }
6072 #[inline(always)]
6073 fn is_sign_negative_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6074 !a.is_sign_positive()
6075 }
6076 #[inline(always)]
6077 fn select_f64x2(self, a: mask64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
6078 crate::kernel!(
6079 #[inline(always)]
6080 fn kernel(
6081 token: Avx2,
6082 a: mask64x2<Avx2>,
6083 b: f64x2<Avx2>,
6084 c: f64x2<Avx2>,
6085 ) -> f64x2<Avx2> {
6086 _mm_blendv_pd(c.into(), b.into(), _mm_castsi128_pd(a.into())).simd_into(token)
6087 }
6088 );
6089 kernel(self, a, b, c)
6090 }
6091 #[inline(always)]
6092 fn combine_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x4<Self> {
6093 crate::kernel!(
6094 #[inline(always)]
6095 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x4<Avx2> {
6096 _mm256_setr_m128d(a.into(), b.into()).simd_into(token)
6097 }
6098 );
6099 kernel(self, a, b)
6100 }
6101 #[inline(always)]
6102 fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2<Self>; 4usize] {
6103 crate::kernel!(
6104 #[inline(always)]
6105 fn kernel(token: Avx2, src: &[f64; 8usize]) -> [f64x2<Avx2>; 4usize] {
6106 let (chunks, []) = src.as_chunks::<4>() else {
6107 unreachable!()
6108 };
6109 let v0: __m256d =
6110 crate::transmute::checked_transmute_copy::<[f64; 4], __m256d>(&chunks[0]);
6111 let v1: __m256d =
6112 crate::transmute::checked_transmute_copy::<[f64; 4], __m256d>(&chunks[1]);
6113 let lo = _mm256_unpacklo_pd(v0, v1);
6114 let hi = _mm256_unpackhi_pd(v0, v1);
6115 let out0 = _mm256_permute2f128_pd::<0x20>(lo, hi);
6116 let out1 = _mm256_permute2f128_pd::<0x31>(lo, hi);
6117 let outputs: [__m128d; 4] = crate::transmute::checked_transmute_copy(&[out0, out1]);
6118 [
6119 outputs[0].simd_into(token),
6120 outputs[1].simd_into(token),
6121 outputs[2].simd_into(token),
6122 outputs[3].simd_into(token),
6123 ]
6124 }
6125 );
6126 kernel(self, src)
6127 }
6128 #[inline(always)]
6129 fn store_four_interleaved_f64x2(
6130 self,
6131 vectors: [f64x2<Self>; 4usize],
6132 dest: &mut [f64; 8usize],
6133 ) -> () {
6134 crate::kernel!(
6135 #[inline(always)]
6136 fn kernel(token: Avx2, vectors: [f64x2<Avx2>; 4usize], dest: &mut [f64; 8usize]) -> () {
6137 let _ = token;
6138 let inputs: [__m128d; 4] = [
6139 vectors[0].into(),
6140 vectors[1].into(),
6141 vectors[2].into(),
6142 vectors[3].into(),
6143 ];
6144 let wide_inputs: [__m256d; 2] = crate::transmute::checked_transmute_copy(&inputs);
6145 let v0 = wide_inputs[0];
6146 let v1 = wide_inputs[1];
6147 let lo = _mm256_permute2f128_pd::<0x20>(v0, v1);
6148 let hi = _mm256_permute2f128_pd::<0x31>(v0, v1);
6149 let out0 = _mm256_unpacklo_pd(lo, hi);
6150 let out1 = _mm256_unpackhi_pd(lo, hi);
6151 let (chunks, []) = dest.as_chunks_mut::<4>() else {
6152 unreachable!()
6153 };
6154 crate::transmute::checked_transmute_store::<__m256d, [f64; 4]>(
6155 out0,
6156 &mut chunks[0],
6157 );
6158 crate::transmute::checked_transmute_store::<__m256d, [f64; 4]>(
6159 out1,
6160 &mut chunks[1],
6161 );
6162 }
6163 );
6164 kernel(self, vectors, dest);
6165 }
6166 #[inline(always)]
6167 fn narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
6168 crate::kernel!(
6169 #[inline(always)]
6170 fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f32x4<Avx2> {
6171 let low = _mm_cvtpd_ps(a.into());
6172 let high = _mm_cvtpd_ps(b.into());
6173 _mm_movelh_ps(low, high).simd_into(token)
6174 }
6175 );
6176 kernel(self, a, b)
6177 }
6178 #[inline(always)]
6179 fn saturating_narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
6180 self.narrow_f64x2(a, b)
6181 }
6182 #[inline(always)]
6183 fn relaxed_narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
6184 self.narrow_f64x2(a, b)
6185 }
6186 #[inline(always)]
6187 fn cvt_u64_f64x2(self, a: f64x2<Self>) -> u64x2<Self> {
6188 crate::kernel!(
6189 #[inline(always)]
6190 fn kernel(token: Avx2, a: f64x2<Avx2>) -> u64x2<Avx2> {
6191 let a = a.into();
6192 let bits = _mm_castpd_si128(a);
6193 let exponent = _mm_srli_epi64::<52>(bits);
6194 let significand = _mm_or_si128(
6195 _mm_and_si128(bits, _mm_set1_epi64x(0x000f_ffff_ffff_ffff)),
6196 _mm_set1_epi64x(0x0010_0000_0000_0000),
6197 );
6198 let shift_bias = _mm_set1_epi64x(1075);
6199 let right_count = _mm_sub_epi64(shift_bias, exponent);
6200 let left_count = _mm_sub_epi64(exponent, shift_bias);
6201 let converted = _mm_or_si128(
6202 _mm_srlv_epi64(significand, right_count),
6203 _mm_sllv_epi64(significand, left_count),
6204 );
6205 converted.simd_into(token)
6206 }
6207 );
6208 kernel(self, a)
6209 }
6210 #[inline(always)]
6211 fn cvt_u64_precise_f64x2(self, a: f64x2<Self>) -> u64x2<Self> {
6212 crate::kernel!(
6213 #[inline(always)]
6214 fn kernel(token: Avx2, a: f64x2<Avx2>) -> u64x2<Avx2> {
6215 let a = a.into();
6216 let bits = _mm_castpd_si128(a);
6217 let exponent = _mm_srli_epi64::<52>(bits);
6218 let significand = _mm_or_si128(
6219 _mm_and_si128(bits, _mm_set1_epi64x(0x000f_ffff_ffff_ffff)),
6220 _mm_set1_epi64x(0x0010_0000_0000_0000),
6221 );
6222 let shift_bias = _mm_set1_epi64x(1075);
6223 let right_count = _mm_sub_epi64(shift_bias, exponent);
6224 let left_count = _mm_sub_epi64(exponent, shift_bias);
6225 let converted = _mm_or_si128(
6226 _mm_srlv_epi64(significand, right_count),
6227 _mm_sllv_epi64(significand, left_count),
6228 );
6229 let overflow =
6230 _mm_castpd_si128(_mm_cmpge_pd(a, _mm_set1_pd(18_446_744_073_709_551_616.0)));
6231 _mm_or_si128(converted, overflow).simd_into(token)
6232 }
6233 );
6234 kernel(self, a)
6235 }
6236 #[inline(always)]
6237 #[cfg(target_arch = "x86_64")]
6238 fn cvt_i64_f64x2(self, a: f64x2<Self>) -> i64x2<Self> {
6239 crate::kernel!(
6240 #[inline(always)]
6241 fn kernel(token: Avx2, a: f64x2<Avx2>) -> i64x2<Avx2> {
6242 let a = a.into();
6243 let low = _mm_cvttsd_si64(a);
6244 let high = _mm_cvttsd_si64(_mm_unpackhi_pd(a, a));
6245 _mm_set_epi64x(high, low).simd_into(token)
6246 }
6247 );
6248 kernel(self, a)
6249 }
6250 #[cfg(target_arch = "x86")]
6251 fn cvt_i64_f64x2(self, a: f64x2<Self>) -> i64x2<Self> {
6252 [a[0usize] as i64, a[1usize] as i64].simd_into(self)
6253 }
6254 #[inline(always)]
6255 fn cvt_i64_precise_f64x2(self, a: f64x2<Self>) -> i64x2<Self> {
6256 crate::kernel!(
6257 #[inline(always)]
6258 fn kernel(token: Avx2, a: f64x2<Avx2>) -> i64x2<Avx2> {
6259 let a = a.into();
6260 let bits = _mm_castpd_si128(a);
6261 let absolute = _mm_and_si128(bits, _mm_set1_epi64x(i64::MAX));
6262 let exponent = _mm_srli_epi64::<52>(absolute);
6263 let significand = _mm_or_si128(
6264 _mm_and_si128(bits, _mm_set1_epi64x(0x000f_ffff_ffff_ffff)),
6265 _mm_set1_epi64x(0x0010_0000_0000_0000),
6266 );
6267 let shift_bias = _mm_set1_epi64x(1075);
6268 let right_count = _mm_sub_epi64(shift_bias, exponent);
6269 let left_count = _mm_sub_epi64(exponent, shift_bias);
6270 let magnitude = _mm_or_si128(
6271 _mm_srlv_epi64(significand, right_count),
6272 _mm_sllv_epi64(significand, left_count),
6273 );
6274 let sign = _mm_cmpgt_epi64(_mm_setzero_si128(), bits);
6275 let converted = _mm_sub_epi64(_mm_xor_si128(magnitude, sign), sign);
6276 let overflow = _mm_castpd_si128(_mm_cmpge_pd(
6277 _mm_castsi128_pd(absolute),
6278 _mm_set1_pd(9_223_372_036_854_775_808.0),
6279 ));
6280 let bound = _mm_xor_si128(_mm_set1_epi64x(i64::MAX), sign);
6281 _mm_blendv_epi8(converted, bound, overflow).simd_into(token)
6282 }
6283 );
6284 kernel(self, a)
6285 }
6286 #[inline(always)]
6287 fn abs_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6288 crate::kernel!(
6289 #[inline(always)]
6290 fn kernel(token: Avx2, a: i64x2<Avx2>) -> i64x2<Avx2> {
6291 let a = a.into();
6292 let mask = _mm_cmpgt_epi64(_mm_setzero_si128(), a);
6293 _mm_sub_epi64(_mm_xor_si128(a, mask), mask).simd_into(token)
6294 }
6295 );
6296 kernel(self, a)
6297 }
6298 #[inline(always)]
6299 fn splat_i64x2(self, val: i64) -> i64x2<Self> {
6300 crate::kernel!(
6301 #[inline(always)]
6302 fn kernel(token: Avx2, val: i64) -> i64x2<Avx2> {
6303 _mm_set1_epi64x(val).simd_into(token)
6304 }
6305 );
6306 kernel(self, val)
6307 }
6308 #[inline(always)]
6309 fn reverse_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6310 let indices: u8x16<Self> =
6311 [8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7].simd_into(self);
6312 self.swizzle_dyn_i64x2(a, indices)
6313 }
6314 #[inline(always)]
6315 fn slide_i64x2<const SHIFT: usize>(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6316 if SHIFT >= 2usize {
6317 return b;
6318 }
6319 let result = dyn_alignr_128(
6320 self,
6321 Bytes::to_bytes(b).val.0,
6322 Bytes::to_bytes(a).val.0,
6323 SHIFT * 8usize,
6324 );
6325 Bytes::from_bytes(u8x16 {
6326 val: crate::support::Aligned128(result),
6327 simd: self,
6328 })
6329 }
6330 #[inline(always)]
6331 fn compress_i64x2(self, values: i64x2<Self>, mask: mask64x2<Self>) -> i64x2<Self> {
6332 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
6333 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0);
6334 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
6335 }
6336 #[inline(always)]
6337 fn compress_merge_i64x2(
6338 self,
6339 values: i64x2<Self>,
6340 mask: mask64x2<Self>,
6341 merge: i64x2<Self>,
6342 ) -> i64x2<Self> {
6343 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
6344 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0);
6345 let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control);
6346 let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0));
6347 Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed))
6348 }
6349 #[inline(always)]
6350 fn expand_i64x2(self, values: i64x2<Self>, mask: mask64x2<Self>) -> i64x2<Self> {
6351 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
6352 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0);
6353 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
6354 }
6355 #[inline(always)]
6356 fn expand_merge_i64x2(
6357 self,
6358 values: i64x2<Self>,
6359 mask: mask64x2<Self>,
6360 merge: i64x2<Self>,
6361 ) -> i64x2<Self> {
6362 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
6363 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0);
6364 let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control));
6365 self.select_i64x2(mask, expanded, merge)
6366 }
6367 #[inline(always)]
6368 fn count_ones_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6369 crate::kernel!(
6370 #[inline(always)]
6371 fn kernel(token: Avx2, a: i64x2<Avx2>) -> i64x2<Avx2> {
6372 let value = a.into();
6373 let nibble_mask = _mm_set1_epi8(0x0f);
6374 let lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4);
6375 let low = _mm_and_si128(value, nibble_mask);
6376 let high = _mm_and_si128(_mm_srli_epi16::<4>(value), nibble_mask);
6377 let byte_counts = _mm_add_epi8(
6378 _mm_shuffle_epi8(lookup, low),
6379 _mm_shuffle_epi8(lookup, high),
6380 );
6381 _mm_sad_epu8(byte_counts, _mm_setzero_si128()).simd_into(token)
6382 }
6383 );
6384 kernel(self, a)
6385 }
6386 #[inline(always)]
6387 fn count_zeros_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6388 self.count_ones_i64x2(self.not_i64x2(a))
6389 }
6390 #[inline(always)]
6391 fn add_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6392 crate::kernel!(
6393 #[inline(always)]
6394 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
6395 _mm_add_epi64(a.into(), b.into()).simd_into(token)
6396 }
6397 );
6398 kernel(self, a, b)
6399 }
6400 #[inline(always)]
6401 fn saturating_add_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6402 crate::kernel!(
6403 #[inline(always)]
6404 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
6405 let a = a.into();
6406 let b = b.into();
6407 let wrapped = _mm_add_epi64(a, b);
6408 let overflow = _mm_xor_si128(_mm_cmpgt_epi64(a, wrapped), b);
6409 let bound = _mm_add_epi64(_mm_srli_epi64::<63>(a), _mm_set1_epi64x(i64::MAX));
6410 let result = _mm_blendv_pd(
6411 _mm_castsi128_pd(wrapped),
6412 _mm_castsi128_pd(bound),
6413 _mm_castsi128_pd(overflow),
6414 );
6415 _mm_castpd_si128(result).simd_into(token)
6416 }
6417 );
6418 kernel(self, a, b)
6419 }
6420 #[inline(always)]
6421 fn sub_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6422 crate::kernel!(
6423 #[inline(always)]
6424 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
6425 _mm_sub_epi64(a.into(), b.into()).simd_into(token)
6426 }
6427 );
6428 kernel(self, a, b)
6429 }
6430 #[inline(always)]
6431 fn saturating_sub_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6432 crate::kernel!(
6433 #[inline(always)]
6434 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
6435 let a = a.into();
6436 let b = b.into();
6437 let wrapped = _mm_sub_epi64(a, b);
6438 let overflow = _mm_xor_si128(_mm_cmpgt_epi64(wrapped, a), b);
6439 let bound = _mm_add_epi64(_mm_srli_epi64::<63>(a), _mm_set1_epi64x(i64::MAX));
6440 let result = _mm_blendv_pd(
6441 _mm_castsi128_pd(wrapped),
6442 _mm_castsi128_pd(bound),
6443 _mm_castsi128_pd(overflow),
6444 );
6445 _mm_castpd_si128(result).simd_into(token)
6446 }
6447 );
6448 kernel(self, a, b)
6449 }
6450 #[inline(always)]
6451 fn mul_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6452 crate::kernel!(
6453 #[inline(always)]
6454 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
6455 let a = a.into();
6456 let b = b.into();
6457 let a_high = _mm_shuffle_epi32::<0xf5>(a);
6458 let b_high = _mm_shuffle_epi32::<0xf5>(b);
6459 let cross = _mm_add_epi64(_mm_mul_epu32(a_high, b), _mm_mul_epu32(a, b_high));
6460 let low = _mm_mul_epu32(a, b);
6461 _mm_add_epi64(low, _mm_slli_epi64::<32>(cross)).simd_into(token)
6462 }
6463 );
6464 kernel(self, a, b)
6465 }
6466 #[inline(always)]
6467 fn and_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6468 crate::kernel!(
6469 #[inline(always)]
6470 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
6471 _mm_and_si128(a.into(), b.into()).simd_into(token)
6472 }
6473 );
6474 kernel(self, a, b)
6475 }
6476 #[inline(always)]
6477 fn or_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6478 crate::kernel!(
6479 #[inline(always)]
6480 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
6481 _mm_or_si128(a.into(), b.into()).simd_into(token)
6482 }
6483 );
6484 kernel(self, a, b)
6485 }
6486 #[inline(always)]
6487 fn xor_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6488 crate::kernel!(
6489 #[inline(always)]
6490 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
6491 _mm_xor_si128(a.into(), b.into()).simd_into(token)
6492 }
6493 );
6494 kernel(self, a, b)
6495 }
6496 #[inline(always)]
6497 fn not_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6498 a ^ !0
6499 }
6500 #[inline(always)]
6501 fn shl_i64x2(self, a: i64x2<Self>, shift: u32) -> i64x2<Self> {
6502 crate::kernel!(
6503 #[inline(always)]
6504 fn kernel(token: Avx2, a: i64x2<Avx2>, shift: u32) -> i64x2<Avx2> {
6505 _mm_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
6506 }
6507 );
6508 kernel(self, a, shift)
6509 }
6510 #[inline(always)]
6511 fn shlv_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6512 crate::kernel!(
6513 #[inline(always)]
6514 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
6515 _mm_sllv_epi64(a.into(), b.into()).simd_into(token)
6516 }
6517 );
6518 kernel(self, a, b)
6519 }
6520 #[inline(always)]
6521 fn shr_i64x2(self, a: i64x2<Self>, shift: u32) -> i64x2<Self> {
6522 crate::kernel!(
6523 #[inline(always)]
6524 fn kernel(token: Avx2, a: i64x2<Avx2>, shift: u32) -> i64x2<Avx2> {
6525 let value = a.into();
6526 let count = _mm_cvtsi32_si128(shift.cast_signed());
6527 let shifted_bias = _mm_srl_epi64(_mm_set1_epi64x(i64::MIN), count);
6528 let shifted = _mm_srl_epi64(value, count);
6529 _mm_sub_epi64(_mm_xor_si128(shifted, shifted_bias), shifted_bias).simd_into(token)
6530 }
6531 );
6532 kernel(self, a, shift)
6533 }
6534 #[inline(always)]
6535 fn shrv_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6536 crate::kernel!(
6537 #[inline(always)]
6538 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
6539 let value = a.into();
6540 let counts = b.into();
6541 let bias = _mm_set1_epi64x(i64::MIN);
6542 let shifted_bias = _mm_srlv_epi64(bias, counts);
6543 let shifted = _mm_srlv_epi64(value, counts);
6544 _mm_sub_epi64(_mm_xor_si128(shifted, shifted_bias), shifted_bias).simd_into(token)
6545 }
6546 );
6547 kernel(self, a, b)
6548 }
6549 #[inline(always)]
6550 fn reduce_max_i64x2(self, a: i64x2<Self>) -> i64 {
6551 crate::kernel!(
6552 #[inline(always)]
6553 fn kernel(token: Avx2, a: i64x2<Avx2>) -> i64 {
6554 let lanes: [i64; 2] = a.into();
6555 lanes[0].max(lanes[1])
6556 }
6557 );
6558 kernel(self, a)
6559 }
6560 #[inline(always)]
6561 fn reduce_min_i64x2(self, a: i64x2<Self>) -> i64 {
6562 crate::kernel!(
6563 #[inline(always)]
6564 fn kernel(token: Avx2, a: i64x2<Avx2>) -> i64 {
6565 let lanes: [i64; 2] = a.into();
6566 lanes[0].min(lanes[1])
6567 }
6568 );
6569 kernel(self, a)
6570 }
6571 #[inline(always)]
6572 fn reduce_sum_i64x2(self, a: i64x2<Self>) -> i64 {
6573 crate::kernel!(
6574 #[inline(always)]
6575 fn kernel(token: Avx2, a: i64x2<Avx2>) -> i64 {
6576 let sum: __m128i = a.into();
6577 let sum = _mm_add_epi64(sum, _mm_srli_si128::<8>(sum));
6578 let lanes: [i64; 2usize] = crate::transmute::checked_transmute_copy(&sum);
6579 lanes[0]
6580 }
6581 );
6582 kernel(self, a)
6583 }
6584 #[inline(always)]
6585 fn reduce_product_i64x2(self, a: i64x2<Self>) -> i64 {
6586 let mul_level_0: [i64; 1usize] = [a[0usize].wrapping_mul(a[1usize])];
6587 mul_level_0[0]
6588 }
6589 #[inline(always)]
6590 fn max_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6591 crate::kernel!(
6592 #[inline(always)]
6593 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
6594 let a = a.into();
6595 let b = b.into();
6596 let mask = _mm_cmpgt_epi64(b, a);
6597 let result = _mm_blendv_pd(
6598 _mm_castsi128_pd(a),
6599 _mm_castsi128_pd(b),
6600 _mm_castsi128_pd(mask),
6601 );
6602 _mm_castpd_si128(result).simd_into(token)
6603 }
6604 );
6605 kernel(self, a, b)
6606 }
6607 #[inline(always)]
6608 fn min_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6609 crate::kernel!(
6610 #[inline(always)]
6611 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
6612 let a = a.into();
6613 let b = b.into();
6614 let mask = _mm_cmpgt_epi64(a, b);
6615 let result = _mm_blendv_pd(
6616 _mm_castsi128_pd(a),
6617 _mm_castsi128_pd(b),
6618 _mm_castsi128_pd(mask),
6619 );
6620 _mm_castpd_si128(result).simd_into(token)
6621 }
6622 );
6623 kernel(self, a, b)
6624 }
6625 #[inline(always)]
6626 fn clamp_i64x2(self, a: i64x2<Self>, b: i64x2<Self>, c: i64x2<Self>) -> i64x2<Self> {
6627 debug_assert!(
6628 b.simd_le(c).all_true(),
6629 "clamp requires min <= max for all elements"
6630 );
6631 a.max(b).min(c)
6632 }
6633 #[inline(always)]
6634 fn simd_eq_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
6635 crate::kernel!(
6636 #[inline(always)]
6637 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> mask64x2<Avx2> {
6638 _mm_cmpeq_epi64(a.into(), b.into()).simd_into(token)
6639 }
6640 );
6641 kernel(self, a, b)
6642 }
6643 #[inline(always)]
6644 fn simd_lt_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
6645 crate::kernel!(
6646 #[inline(always)]
6647 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> mask64x2<Avx2> {
6648 _mm_cmpgt_epi64(b.into(), a.into()).simd_into(token)
6649 }
6650 );
6651 kernel(self, a, b)
6652 }
6653 #[inline(always)]
6654 fn simd_le_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
6655 crate::kernel!(
6656 #[inline(always)]
6657 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> mask64x2<Avx2> {
6658 _mm_xor_si128(_mm_cmpgt_epi64(a.into(), b.into()), _mm_set1_epi64x(-1))
6659 .simd_into(token)
6660 }
6661 );
6662 kernel(self, a, b)
6663 }
6664 #[inline(always)]
6665 fn zip_low_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6666 crate::kernel!(
6667 #[inline(always)]
6668 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
6669 _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token)
6670 }
6671 );
6672 kernel(self, a, b)
6673 }
6674 #[inline(always)]
6675 fn zip_high_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6676 crate::kernel!(
6677 #[inline(always)]
6678 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
6679 _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token)
6680 }
6681 );
6682 kernel(self, a, b)
6683 }
6684 #[inline(always)]
6685 fn unzip_low_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6686 crate::kernel!(
6687 #[inline(always)]
6688 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
6689 _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token)
6690 }
6691 );
6692 kernel(self, a, b)
6693 }
6694 #[inline(always)]
6695 fn unzip_high_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6696 crate::kernel!(
6697 #[inline(always)]
6698 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
6699 _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token)
6700 }
6701 );
6702 kernel(self, a, b)
6703 }
6704 #[inline(always)]
6705 fn interleave_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> (i64x2<Self>, i64x2<Self>) {
6706 (self.zip_low_i64x2(a, b), self.zip_high_i64x2(a, b))
6707 }
6708 #[inline(always)]
6709 fn deinterleave_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> (i64x2<Self>, i64x2<Self>) {
6710 (self.unzip_low_i64x2(a, b), self.unzip_high_i64x2(a, b))
6711 }
6712 #[inline(always)]
6713 fn select_i64x2(self, a: mask64x2<Self>, b: i64x2<Self>, c: i64x2<Self>) -> i64x2<Self> {
6714 crate::kernel!(
6715 #[inline(always)]
6716 fn kernel(
6717 token: Avx2,
6718 a: mask64x2<Avx2>,
6719 b: i64x2<Avx2>,
6720 c: i64x2<Avx2>,
6721 ) -> i64x2<Avx2> {
6722 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
6723 }
6724 );
6725 kernel(self, a, b, c)
6726 }
6727 #[inline(always)]
6728 fn combine_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x4<Self> {
6729 crate::kernel!(
6730 #[inline(always)]
6731 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x4<Avx2> {
6732 _mm256_setr_m128i(a.into(), b.into()).simd_into(token)
6733 }
6734 );
6735 kernel(self, a, b)
6736 }
6737 #[inline(always)]
6738 fn neg_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6739 crate::kernel!(
6740 #[inline(always)]
6741 fn kernel(token: Avx2, a: i64x2<Avx2>) -> i64x2<Avx2> {
6742 _mm_sub_epi64(_mm_setzero_si128(), a.into()).simd_into(token)
6743 }
6744 );
6745 kernel(self, a)
6746 }
6747 #[inline(always)]
6748 fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2<Self>; 4usize] {
6749 crate::kernel!(
6750 #[inline(always)]
6751 fn kernel(token: Avx2, src: &[i64; 8usize]) -> [i64x2<Avx2>; 4usize] {
6752 let (chunks, []) = src.as_chunks::<4>() else {
6753 unreachable!()
6754 };
6755 let v0: __m256i =
6756 crate::transmute::checked_transmute_copy::<[i64; 4], __m256i>(&chunks[0]);
6757 let v1: __m256i =
6758 crate::transmute::checked_transmute_copy::<[i64; 4], __m256i>(&chunks[1]);
6759 let lo = _mm256_unpacklo_epi64(v0, v1);
6760 let hi = _mm256_unpackhi_epi64(v0, v1);
6761 let out0 = _mm256_permute2x128_si256::<0x20>(lo, hi);
6762 let out1 = _mm256_permute2x128_si256::<0x31>(lo, hi);
6763 let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&[out0, out1]);
6764 [
6765 outputs[0].simd_into(token),
6766 outputs[1].simd_into(token),
6767 outputs[2].simd_into(token),
6768 outputs[3].simd_into(token),
6769 ]
6770 }
6771 );
6772 kernel(self, src)
6773 }
6774 #[inline(always)]
6775 fn store_four_interleaved_i64x2(
6776 self,
6777 vectors: [i64x2<Self>; 4usize],
6778 dest: &mut [i64; 8usize],
6779 ) -> () {
6780 crate::kernel!(
6781 #[inline(always)]
6782 fn kernel(token: Avx2, vectors: [i64x2<Avx2>; 4usize], dest: &mut [i64; 8usize]) -> () {
6783 let _ = token;
6784 let inputs: [__m128i; 4] = [
6785 vectors[0].into(),
6786 vectors[1].into(),
6787 vectors[2].into(),
6788 vectors[3].into(),
6789 ];
6790 let wide_inputs: [__m256i; 2] = crate::transmute::checked_transmute_copy(&inputs);
6791 let v0 = wide_inputs[0];
6792 let v1 = wide_inputs[1];
6793 let lo = _mm256_permute2x128_si256::<0x20>(v0, v1);
6794 let hi = _mm256_permute2x128_si256::<0x31>(v0, v1);
6795 let out0 = _mm256_unpacklo_epi64(lo, hi);
6796 let out1 = _mm256_unpackhi_epi64(lo, hi);
6797 let (chunks, []) = dest.as_chunks_mut::<4>() else {
6798 unreachable!()
6799 };
6800 crate::transmute::checked_transmute_store::<__m256i, [i64; 4]>(
6801 out0,
6802 &mut chunks[0],
6803 );
6804 crate::transmute::checked_transmute_store::<__m256i, [i64; 4]>(
6805 out1,
6806 &mut chunks[1],
6807 );
6808 }
6809 );
6810 kernel(self, vectors, dest);
6811 }
6812 #[inline(always)]
6813 fn narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
6814 crate::kernel!(
6815 #[inline(always)]
6816 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i32x4<Avx2> {
6817 _mm_castps_si128(_mm_shuffle_ps::<0x88>(
6818 _mm_castsi128_ps(a.into()),
6819 _mm_castsi128_ps(b.into()),
6820 ))
6821 .simd_into(token)
6822 }
6823 );
6824 kernel(self, a, b)
6825 }
6826 #[inline(always)]
6827 fn saturating_narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
6828 crate::kernel!(
6829 #[inline(always)]
6830 fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i32x4<Avx2> {
6831 let a = a.into();
6832 let b = b.into();
6833 let low = _mm_castps_si128(_mm_shuffle_ps::<0x88>(
6834 _mm_castsi128_ps(a),
6835 _mm_castsi128_ps(b),
6836 ));
6837 let high = _mm_castps_si128(_mm_shuffle_ps::<0xdd>(
6838 _mm_castsi128_ps(a),
6839 _mm_castsi128_ps(b),
6840 ));
6841 let low_sign = _mm_srai_epi32::<31>(low);
6842 let fits = _mm_cmpeq_epi32(high, low_sign);
6843 let high_sign = _mm_srai_epi32::<31>(high);
6844 let bound = _mm_xor_si128(high_sign, _mm_set1_epi32(i32::MAX));
6845 _mm_blendv_epi8(bound, low, fits).simd_into(token)
6846 }
6847 );
6848 kernel(self, a, b)
6849 }
6850 #[inline(always)]
6851 fn relaxed_narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
6852 debug_assert!(
6853 a.as_slice()
6854 .iter()
6855 .chain(b.as_slice())
6856 .all(|&value| { value >= i32::MIN as i64 && value <= i32::MAX as i64 }),
6857 "relaxed_narrow inputs must fit in the destination type",
6858 );
6859 self.narrow_i64x2(a, b)
6860 }
6861 #[inline(always)]
6862 fn cvt_f64_i64x2(self, a: i64x2<Self>) -> f64x2<Self> {
6863 crate::kernel!(
6864 #[inline(always)]
6865 fn kernel(token: Avx2, a: i64x2<Avx2>) -> f64x2<Avx2> {
6866 let a = a.into();
6867 let low = _mm_blend_epi16::<0xcc>(a, _mm_set1_epi64x(0x4330_0000_0000_0000));
6868 let high = _mm_srli_epi64::<32>(a);
6869 let high = _mm_xor_si128(high, _mm_set1_epi64x(4985484789646622720i64));
6870 let high = _mm_sub_pd(
6871 _mm_castsi128_pd(high),
6872 _mm_set1_pd(f64::from_bits(4985484789647671296u64)),
6873 );
6874 _mm_add_pd(_mm_castsi128_pd(low), high).simd_into(token)
6875 }
6876 );
6877 kernel(self, a)
6878 }
6879 #[inline(always)]
6880 fn splat_u64x2(self, val: u64) -> u64x2<Self> {
6881 crate::kernel!(
6882 #[inline(always)]
6883 fn kernel(token: Avx2, val: u64) -> u64x2<Avx2> {
6884 _mm_set1_epi64x(val.cast_signed()).simd_into(token)
6885 }
6886 );
6887 kernel(self, val)
6888 }
6889 #[inline(always)]
6890 fn reverse_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
6891 let indices: u8x16<Self> =
6892 [8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7].simd_into(self);
6893 self.swizzle_dyn_u64x2(a, indices)
6894 }
6895 #[inline(always)]
6896 fn slide_u64x2<const SHIFT: usize>(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
6897 if SHIFT >= 2usize {
6898 return b;
6899 }
6900 let result = dyn_alignr_128(
6901 self,
6902 Bytes::to_bytes(b).val.0,
6903 Bytes::to_bytes(a).val.0,
6904 SHIFT * 8usize,
6905 );
6906 Bytes::from_bytes(u8x16 {
6907 val: crate::support::Aligned128(result),
6908 simd: self,
6909 })
6910 }
6911 #[inline(always)]
6912 fn compress_u64x2(self, values: u64x2<Self>, mask: mask64x2<Self>) -> u64x2<Self> {
6913 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
6914 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0);
6915 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
6916 }
6917 #[inline(always)]
6918 fn compress_merge_u64x2(
6919 self,
6920 values: u64x2<Self>,
6921 mask: mask64x2<Self>,
6922 merge: u64x2<Self>,
6923 ) -> u64x2<Self> {
6924 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
6925 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0);
6926 let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control);
6927 let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0));
6928 Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed))
6929 }
6930 #[inline(always)]
6931 fn expand_u64x2(self, values: u64x2<Self>, mask: mask64x2<Self>) -> u64x2<Self> {
6932 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
6933 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0);
6934 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
6935 }
6936 #[inline(always)]
6937 fn expand_merge_u64x2(
6938 self,
6939 values: u64x2<Self>,
6940 mask: mask64x2<Self>,
6941 merge: u64x2<Self>,
6942 ) -> u64x2<Self> {
6943 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
6944 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0);
6945 let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control));
6946 self.select_u64x2(mask, expanded, merge)
6947 }
6948 #[inline(always)]
6949 fn count_ones_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
6950 crate::kernel!(
6951 #[inline(always)]
6952 fn kernel(token: Avx2, a: u64x2<Avx2>) -> u64x2<Avx2> {
6953 let value = a.into();
6954 let nibble_mask = _mm_set1_epi8(0x0f);
6955 let lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4);
6956 let low = _mm_and_si128(value, nibble_mask);
6957 let high = _mm_and_si128(_mm_srli_epi16::<4>(value), nibble_mask);
6958 let byte_counts = _mm_add_epi8(
6959 _mm_shuffle_epi8(lookup, low),
6960 _mm_shuffle_epi8(lookup, high),
6961 );
6962 _mm_sad_epu8(byte_counts, _mm_setzero_si128()).simd_into(token)
6963 }
6964 );
6965 kernel(self, a)
6966 }
6967 #[inline(always)]
6968 fn count_zeros_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
6969 self.count_ones_u64x2(self.not_u64x2(a))
6970 }
6971 #[inline(always)]
6972 fn add_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
6973 crate::kernel!(
6974 #[inline(always)]
6975 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
6976 _mm_add_epi64(a.into(), b.into()).simd_into(token)
6977 }
6978 );
6979 kernel(self, a, b)
6980 }
6981 #[inline(always)]
6982 fn saturating_add_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
6983 crate::kernel!(
6984 #[inline(always)]
6985 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
6986 let a = a.into();
6987 let b = b.into();
6988 let wrapped = _mm_add_epi64(a, b);
6989 let sign_bias = _mm_set1_epi64x(i64::MIN);
6990 let overflow = _mm_cmpgt_epi64(
6991 _mm_xor_si128(a, sign_bias),
6992 _mm_xor_si128(wrapped, sign_bias),
6993 );
6994 _mm_or_si128(wrapped, overflow).simd_into(token)
6995 }
6996 );
6997 kernel(self, a, b)
6998 }
6999 #[inline(always)]
7000 fn sub_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7001 crate::kernel!(
7002 #[inline(always)]
7003 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
7004 _mm_sub_epi64(a.into(), b.into()).simd_into(token)
7005 }
7006 );
7007 kernel(self, a, b)
7008 }
7009 #[inline(always)]
7010 fn saturating_sub_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7011 crate::kernel!(
7012 #[inline(always)]
7013 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
7014 let a = a.into();
7015 let b = b.into();
7016 let wrapped = _mm_sub_epi64(a, b);
7017 let sign_bias = _mm_set1_epi64x(i64::MIN);
7018 let no_borrow =
7019 _mm_cmpgt_epi64(_mm_xor_si128(a, sign_bias), _mm_xor_si128(b, sign_bias));
7020 _mm_and_si128(wrapped, no_borrow).simd_into(token)
7021 }
7022 );
7023 kernel(self, a, b)
7024 }
7025 #[inline(always)]
7026 fn mul_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7027 crate::kernel!(
7028 #[inline(always)]
7029 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
7030 let a = a.into();
7031 let b = b.into();
7032 let a_high = _mm_shuffle_epi32::<0xf5>(a);
7033 let b_high = _mm_shuffle_epi32::<0xf5>(b);
7034 let cross = _mm_add_epi64(_mm_mul_epu32(a_high, b), _mm_mul_epu32(a, b_high));
7035 let low = _mm_mul_epu32(a, b);
7036 _mm_add_epi64(low, _mm_slli_epi64::<32>(cross)).simd_into(token)
7037 }
7038 );
7039 kernel(self, a, b)
7040 }
7041 #[inline(always)]
7042 fn and_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7043 crate::kernel!(
7044 #[inline(always)]
7045 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
7046 _mm_and_si128(a.into(), b.into()).simd_into(token)
7047 }
7048 );
7049 kernel(self, a, b)
7050 }
7051 #[inline(always)]
7052 fn or_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7053 crate::kernel!(
7054 #[inline(always)]
7055 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
7056 _mm_or_si128(a.into(), b.into()).simd_into(token)
7057 }
7058 );
7059 kernel(self, a, b)
7060 }
7061 #[inline(always)]
7062 fn xor_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7063 crate::kernel!(
7064 #[inline(always)]
7065 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
7066 _mm_xor_si128(a.into(), b.into()).simd_into(token)
7067 }
7068 );
7069 kernel(self, a, b)
7070 }
7071 #[inline(always)]
7072 fn not_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
7073 a ^ !0
7074 }
7075 #[inline(always)]
7076 fn shl_u64x2(self, a: u64x2<Self>, shift: u32) -> u64x2<Self> {
7077 crate::kernel!(
7078 #[inline(always)]
7079 fn kernel(token: Avx2, a: u64x2<Avx2>, shift: u32) -> u64x2<Avx2> {
7080 _mm_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
7081 }
7082 );
7083 kernel(self, a, shift)
7084 }
7085 #[inline(always)]
7086 fn shlv_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7087 crate::kernel!(
7088 #[inline(always)]
7089 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
7090 _mm_sllv_epi64(a.into(), b.into()).simd_into(token)
7091 }
7092 );
7093 kernel(self, a, b)
7094 }
7095 #[inline(always)]
7096 fn shr_u64x2(self, a: u64x2<Self>, shift: u32) -> u64x2<Self> {
7097 crate::kernel!(
7098 #[inline(always)]
7099 fn kernel(token: Avx2, a: u64x2<Avx2>, shift: u32) -> u64x2<Avx2> {
7100 _mm_srl_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
7101 }
7102 );
7103 kernel(self, a, shift)
7104 }
7105 #[inline(always)]
7106 fn shrv_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7107 crate::kernel!(
7108 #[inline(always)]
7109 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
7110 _mm_srlv_epi64(a.into(), b.into()).simd_into(token)
7111 }
7112 );
7113 kernel(self, a, b)
7114 }
7115 #[inline(always)]
7116 fn reduce_max_u64x2(self, a: u64x2<Self>) -> u64 {
7117 crate::kernel!(
7118 #[inline(always)]
7119 fn kernel(token: Avx2, a: u64x2<Avx2>) -> u64 {
7120 let lanes: [u64; 2] = a.into();
7121 lanes[0].max(lanes[1])
7122 }
7123 );
7124 kernel(self, a)
7125 }
7126 #[inline(always)]
7127 fn reduce_min_u64x2(self, a: u64x2<Self>) -> u64 {
7128 crate::kernel!(
7129 #[inline(always)]
7130 fn kernel(token: Avx2, a: u64x2<Avx2>) -> u64 {
7131 let lanes: [u64; 2] = a.into();
7132 lanes[0].min(lanes[1])
7133 }
7134 );
7135 kernel(self, a)
7136 }
7137 #[inline(always)]
7138 fn reduce_sum_u64x2(self, a: u64x2<Self>) -> u64 {
7139 crate::kernel!(
7140 #[inline(always)]
7141 fn kernel(token: Avx2, a: u64x2<Avx2>) -> u64 {
7142 let sum: __m128i = a.into();
7143 let sum = _mm_add_epi64(sum, _mm_srli_si128::<8>(sum));
7144 let lanes: [u64; 2usize] = crate::transmute::checked_transmute_copy(&sum);
7145 lanes[0]
7146 }
7147 );
7148 kernel(self, a)
7149 }
7150 #[inline(always)]
7151 fn reduce_product_u64x2(self, a: u64x2<Self>) -> u64 {
7152 let mul_level_0: [u64; 1usize] = [a[0usize].wrapping_mul(a[1usize])];
7153 mul_level_0[0]
7154 }
7155 #[inline(always)]
7156 fn max_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7157 crate::kernel!(
7158 #[inline(always)]
7159 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
7160 let a = a.into();
7161 let b = b.into();
7162 let mask = {
7163 let sign_bit = _mm_set1_epi64x(0x8000000000000000u64.cast_signed());
7164 let lhs_signed = _mm_xor_si128(b, sign_bit);
7165 let rhs_signed = _mm_xor_si128(a, sign_bit);
7166 _mm_cmpgt_epi64(lhs_signed, rhs_signed)
7167 };
7168 let result = _mm_blendv_pd(
7169 _mm_castsi128_pd(a),
7170 _mm_castsi128_pd(b),
7171 _mm_castsi128_pd(mask),
7172 );
7173 _mm_castpd_si128(result).simd_into(token)
7174 }
7175 );
7176 kernel(self, a, b)
7177 }
7178 #[inline(always)]
7179 fn min_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7180 crate::kernel!(
7181 #[inline(always)]
7182 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
7183 let a = a.into();
7184 let b = b.into();
7185 let mask = {
7186 let sign_bit = _mm_set1_epi64x(0x8000000000000000u64.cast_signed());
7187 let lhs_signed = _mm_xor_si128(a, sign_bit);
7188 let rhs_signed = _mm_xor_si128(b, sign_bit);
7189 _mm_cmpgt_epi64(lhs_signed, rhs_signed)
7190 };
7191 let result = _mm_blendv_pd(
7192 _mm_castsi128_pd(a),
7193 _mm_castsi128_pd(b),
7194 _mm_castsi128_pd(mask),
7195 );
7196 _mm_castpd_si128(result).simd_into(token)
7197 }
7198 );
7199 kernel(self, a, b)
7200 }
7201 #[inline(always)]
7202 fn clamp_u64x2(self, a: u64x2<Self>, b: u64x2<Self>, c: u64x2<Self>) -> u64x2<Self> {
7203 debug_assert!(
7204 b.simd_le(c).all_true(),
7205 "clamp requires min <= max for all elements"
7206 );
7207 a.max(b).min(c)
7208 }
7209 #[inline(always)]
7210 fn simd_eq_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
7211 crate::kernel!(
7212 #[inline(always)]
7213 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> mask64x2<Avx2> {
7214 _mm_cmpeq_epi64(a.into(), b.into()).simd_into(token)
7215 }
7216 );
7217 kernel(self, a, b)
7218 }
7219 #[inline(always)]
7220 fn simd_lt_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
7221 crate::kernel!(
7222 #[inline(always)]
7223 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> mask64x2<Avx2> {
7224 {
7225 let sign_bit = _mm_set1_epi64x(0x8000000000000000u64.cast_signed());
7226 let lhs_signed = _mm_xor_si128(b.into(), sign_bit);
7227 let rhs_signed = _mm_xor_si128(a.into(), sign_bit);
7228 _mm_cmpgt_epi64(lhs_signed, rhs_signed)
7229 }
7230 .simd_into(token)
7231 }
7232 );
7233 kernel(self, a, b)
7234 }
7235 #[inline(always)]
7236 fn simd_le_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
7237 crate::kernel!(
7238 #[inline(always)]
7239 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> mask64x2<Avx2> {
7240 _mm_xor_si128(
7241 {
7242 let sign_bit = _mm_set1_epi64x(0x8000000000000000u64.cast_signed());
7243 let lhs_signed = _mm_xor_si128(a.into(), sign_bit);
7244 let rhs_signed = _mm_xor_si128(b.into(), sign_bit);
7245 _mm_cmpgt_epi64(lhs_signed, rhs_signed)
7246 },
7247 _mm_set1_epi64x(-1),
7248 )
7249 .simd_into(token)
7250 }
7251 );
7252 kernel(self, a, b)
7253 }
7254 #[inline(always)]
7255 fn zip_low_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7256 crate::kernel!(
7257 #[inline(always)]
7258 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
7259 _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token)
7260 }
7261 );
7262 kernel(self, a, b)
7263 }
7264 #[inline(always)]
7265 fn zip_high_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7266 crate::kernel!(
7267 #[inline(always)]
7268 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
7269 _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token)
7270 }
7271 );
7272 kernel(self, a, b)
7273 }
7274 #[inline(always)]
7275 fn unzip_low_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7276 crate::kernel!(
7277 #[inline(always)]
7278 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
7279 _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token)
7280 }
7281 );
7282 kernel(self, a, b)
7283 }
7284 #[inline(always)]
7285 fn unzip_high_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7286 crate::kernel!(
7287 #[inline(always)]
7288 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
7289 _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token)
7290 }
7291 );
7292 kernel(self, a, b)
7293 }
7294 #[inline(always)]
7295 fn interleave_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> (u64x2<Self>, u64x2<Self>) {
7296 (self.zip_low_u64x2(a, b), self.zip_high_u64x2(a, b))
7297 }
7298 #[inline(always)]
7299 fn deinterleave_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> (u64x2<Self>, u64x2<Self>) {
7300 (self.unzip_low_u64x2(a, b), self.unzip_high_u64x2(a, b))
7301 }
7302 #[inline(always)]
7303 fn select_u64x2(self, a: mask64x2<Self>, b: u64x2<Self>, c: u64x2<Self>) -> u64x2<Self> {
7304 crate::kernel!(
7305 #[inline(always)]
7306 fn kernel(
7307 token: Avx2,
7308 a: mask64x2<Avx2>,
7309 b: u64x2<Avx2>,
7310 c: u64x2<Avx2>,
7311 ) -> u64x2<Avx2> {
7312 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
7313 }
7314 );
7315 kernel(self, a, b, c)
7316 }
7317 #[inline(always)]
7318 fn combine_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x4<Self> {
7319 crate::kernel!(
7320 #[inline(always)]
7321 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x4<Avx2> {
7322 _mm256_setr_m128i(a.into(), b.into()).simd_into(token)
7323 }
7324 );
7325 kernel(self, a, b)
7326 }
7327 #[inline(always)]
7328 fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2<Self>; 4usize] {
7329 crate::kernel!(
7330 #[inline(always)]
7331 fn kernel(token: Avx2, src: &[u64; 8usize]) -> [u64x2<Avx2>; 4usize] {
7332 let (chunks, []) = src.as_chunks::<4>() else {
7333 unreachable!()
7334 };
7335 let v0: __m256i =
7336 crate::transmute::checked_transmute_copy::<[u64; 4], __m256i>(&chunks[0]);
7337 let v1: __m256i =
7338 crate::transmute::checked_transmute_copy::<[u64; 4], __m256i>(&chunks[1]);
7339 let lo = _mm256_unpacklo_epi64(v0, v1);
7340 let hi = _mm256_unpackhi_epi64(v0, v1);
7341 let out0 = _mm256_permute2x128_si256::<0x20>(lo, hi);
7342 let out1 = _mm256_permute2x128_si256::<0x31>(lo, hi);
7343 let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&[out0, out1]);
7344 [
7345 outputs[0].simd_into(token),
7346 outputs[1].simd_into(token),
7347 outputs[2].simd_into(token),
7348 outputs[3].simd_into(token),
7349 ]
7350 }
7351 );
7352 kernel(self, src)
7353 }
7354 #[inline(always)]
7355 fn store_four_interleaved_u64x2(
7356 self,
7357 vectors: [u64x2<Self>; 4usize],
7358 dest: &mut [u64; 8usize],
7359 ) -> () {
7360 crate::kernel!(
7361 #[inline(always)]
7362 fn kernel(token: Avx2, vectors: [u64x2<Avx2>; 4usize], dest: &mut [u64; 8usize]) -> () {
7363 let _ = token;
7364 let inputs: [__m128i; 4] = [
7365 vectors[0].into(),
7366 vectors[1].into(),
7367 vectors[2].into(),
7368 vectors[3].into(),
7369 ];
7370 let wide_inputs: [__m256i; 2] = crate::transmute::checked_transmute_copy(&inputs);
7371 let v0 = wide_inputs[0];
7372 let v1 = wide_inputs[1];
7373 let lo = _mm256_permute2x128_si256::<0x20>(v0, v1);
7374 let hi = _mm256_permute2x128_si256::<0x31>(v0, v1);
7375 let out0 = _mm256_unpacklo_epi64(lo, hi);
7376 let out1 = _mm256_unpackhi_epi64(lo, hi);
7377 let (chunks, []) = dest.as_chunks_mut::<4>() else {
7378 unreachable!()
7379 };
7380 crate::transmute::checked_transmute_store::<__m256i, [u64; 4]>(
7381 out0,
7382 &mut chunks[0],
7383 );
7384 crate::transmute::checked_transmute_store::<__m256i, [u64; 4]>(
7385 out1,
7386 &mut chunks[1],
7387 );
7388 }
7389 );
7390 kernel(self, vectors, dest);
7391 }
7392 #[inline(always)]
7393 fn narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
7394 crate::kernel!(
7395 #[inline(always)]
7396 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u32x4<Avx2> {
7397 _mm_castps_si128(_mm_shuffle_ps::<0x88>(
7398 _mm_castsi128_ps(a.into()),
7399 _mm_castsi128_ps(b.into()),
7400 ))
7401 .simd_into(token)
7402 }
7403 );
7404 kernel(self, a, b)
7405 }
7406 #[inline(always)]
7407 fn saturating_narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
7408 crate::kernel!(
7409 #[inline(always)]
7410 fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u32x4<Avx2> {
7411 let a = a.into();
7412 let b = b.into();
7413 let low = _mm_castps_si128(_mm_shuffle_ps::<0x88>(
7414 _mm_castsi128_ps(a),
7415 _mm_castsi128_ps(b),
7416 ));
7417 let high = _mm_castps_si128(_mm_shuffle_ps::<0xdd>(
7418 _mm_castsi128_ps(a),
7419 _mm_castsi128_ps(b),
7420 ));
7421 let zero = _mm_setzero_si128();
7422 let fits = _mm_cmpeq_epi32(high, zero);
7423 let ones = _mm_cmpeq_epi32(zero, zero);
7424 _mm_or_si128(low, _mm_xor_si128(fits, ones)).simd_into(token)
7425 }
7426 );
7427 kernel(self, a, b)
7428 }
7429 #[inline(always)]
7430 fn relaxed_narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
7431 debug_assert!(
7432 a.as_slice()
7433 .iter()
7434 .chain(b.as_slice())
7435 .all(|&value| { value >= u32::MIN as u64 && value <= u32::MAX as u64 }),
7436 "relaxed_narrow inputs must fit in the destination type",
7437 );
7438 self.narrow_u64x2(a, b)
7439 }
7440 #[inline(always)]
7441 fn cvt_f64_u64x2(self, a: u64x2<Self>) -> f64x2<Self> {
7442 crate::kernel!(
7443 #[inline(always)]
7444 fn kernel(token: Avx2, a: u64x2<Avx2>) -> f64x2<Avx2> {
7445 let a = a.into();
7446 let low = _mm_blend_epi16::<0xcc>(a, _mm_set1_epi64x(0x4330_0000_0000_0000));
7447 let high = _mm_srli_epi64::<32>(a);
7448 let high = _mm_xor_si128(high, _mm_set1_epi64x(4985484787499139072i64));
7449 let high = _mm_sub_pd(
7450 _mm_castsi128_pd(high),
7451 _mm_set1_pd(f64::from_bits(4985484787500187648u64)),
7452 );
7453 _mm_add_pd(_mm_castsi128_pd(low), high).simd_into(token)
7454 }
7455 );
7456 kernel(self, a)
7457 }
7458 #[inline(always)]
7459 fn splat_mask64x2(self, val: bool) -> mask64x2<Self> {
7460 crate::kernel!(
7461 #[inline(always)]
7462 fn kernel(token: Avx2, val: bool) -> mask64x2<Avx2> {
7463 let val: i64 = if val { !0 } else { 0 };
7464 _mm_set1_epi64x(val).simd_into(token)
7465 }
7466 );
7467 kernel(self, val)
7468 }
7469 #[inline(always)]
7470 fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2<Self> {
7471 crate::kernel!(
7472 #[inline(always)]
7473 fn kernel(token: Avx2, bits: u64) -> mask64x2<Avx2> {
7474 {
7475 let bit_lanes = _mm_set1_epi64x(bits.cast_signed());
7476 let bit_mask = _mm_set_epi64x(2, 1);
7477 _mm_cmpeq_epi64(_mm_and_si128(bit_lanes, bit_mask), bit_mask)
7478 }
7479 .simd_into(token)
7480 }
7481 );
7482 kernel(self, bits)
7483 }
7484 #[inline(always)]
7485 fn to_bitmask_mask64x2(self, a: mask64x2<Self>) -> u64 {
7486 crate::kernel!(
7487 #[inline(always)]
7488 fn kernel(token: Avx2, a: mask64x2<Avx2>) -> u64 {
7489 _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 as u64
7490 }
7491 );
7492 kernel(self, a)
7493 }
7494 #[inline(always)]
7495 fn from_vector_mask64x2(self, vector: i64x2<Self>) -> mask64x2<Self> {
7496 mask64x2 {
7497 val: vector.val,
7498 simd: self,
7499 }
7500 }
7501 #[inline(always)]
7502 fn to_vector_mask64x2(self, a: mask64x2<Self>) -> i64x2<Self> {
7503 i64x2 {
7504 val: a.val,
7505 simd: self,
7506 }
7507 }
7508 #[inline(always)]
7509 fn set_mask64x2(self, a: &mut mask64x2<Self>, index: usize, value: bool) -> () {
7510 assert!(
7511 index < 2usize,
7512 "mask lane index {index} is out of bounds for {} lanes",
7513 2usize
7514 );
7515 let mut lanes: [i64; 2usize] = (*a).into();
7516 lanes[index] = if value { !0 } else { 0 };
7517 *a = lanes.simd_into(self);
7518 }
7519 #[inline(always)]
7520 fn rotate_elements_left_mask64x2<const OFFSET: usize>(
7521 self,
7522 a: mask64x2<Self>,
7523 ) -> mask64x2<Self> {
7524 let int = i64x2 {
7525 val: crate::transmute::checked_transmute_copy(&a.val),
7526 simd: self,
7527 };
7528 let rotated = int.rotate_elements_left::<OFFSET>();
7529 mask64x2 {
7530 val: crate::transmute::checked_transmute_copy(&rotated.val),
7531 simd: self,
7532 }
7533 }
7534 #[inline(always)]
7535 fn rotate_elements_right_mask64x2<const OFFSET: usize>(
7536 self,
7537 a: mask64x2<Self>,
7538 ) -> mask64x2<Self> {
7539 let int = i64x2 {
7540 val: crate::transmute::checked_transmute_copy(&a.val),
7541 simd: self,
7542 };
7543 let rotated = int.rotate_elements_right::<OFFSET>();
7544 mask64x2 {
7545 val: crate::transmute::checked_transmute_copy(&rotated.val),
7546 simd: self,
7547 }
7548 }
7549 #[inline(always)]
7550 fn and_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7551 crate::kernel!(
7552 #[inline(always)]
7553 fn kernel(token: Avx2, a: mask64x2<Avx2>, b: mask64x2<Avx2>) -> mask64x2<Avx2> {
7554 _mm_and_si128(a.into(), b.into()).simd_into(token)
7555 }
7556 );
7557 kernel(self, a, b)
7558 }
7559 #[inline(always)]
7560 fn or_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7561 crate::kernel!(
7562 #[inline(always)]
7563 fn kernel(token: Avx2, a: mask64x2<Avx2>, b: mask64x2<Avx2>) -> mask64x2<Avx2> {
7564 _mm_or_si128(a.into(), b.into()).simd_into(token)
7565 }
7566 );
7567 kernel(self, a, b)
7568 }
7569 #[inline(always)]
7570 fn xor_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7571 crate::kernel!(
7572 #[inline(always)]
7573 fn kernel(token: Avx2, a: mask64x2<Avx2>, b: mask64x2<Avx2>) -> mask64x2<Avx2> {
7574 _mm_xor_si128(a.into(), b.into()).simd_into(token)
7575 }
7576 );
7577 kernel(self, a, b)
7578 }
7579 #[inline(always)]
7580 fn not_mask64x2(self, a: mask64x2<Self>) -> mask64x2<Self> {
7581 self.xor_mask64x2(a, self.splat_mask64x2(true))
7582 }
7583 #[inline(always)]
7584 fn reverse_mask64x2(self, a: mask64x2<Self>) -> mask64x2<Self> {
7585 let lanes = i64x2 {
7586 val: crate::transmute::checked_transmute_copy(&a.val),
7587 simd: self,
7588 };
7589 let reversed = self.reverse_i64x2(lanes);
7590 mask64x2 {
7591 val: crate::transmute::checked_transmute_copy(&reversed.val),
7592 simd: self,
7593 }
7594 }
7595 #[inline(always)]
7596 fn select_mask64x2(
7597 self,
7598 a: mask64x2<Self>,
7599 b: mask64x2<Self>,
7600 c: mask64x2<Self>,
7601 ) -> mask64x2<Self> {
7602 crate::kernel!(
7603 #[inline(always)]
7604 fn kernel(
7605 token: Avx2,
7606 a: mask64x2<Avx2>,
7607 b: mask64x2<Avx2>,
7608 c: mask64x2<Avx2>,
7609 ) -> mask64x2<Avx2> {
7610 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
7611 }
7612 );
7613 kernel(self, a, b, c)
7614 }
7615 #[inline(always)]
7616 fn simd_eq_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7617 crate::kernel!(
7618 #[inline(always)]
7619 fn kernel(token: Avx2, a: mask64x2<Avx2>, b: mask64x2<Avx2>) -> mask64x2<Avx2> {
7620 _mm_cmpeq_epi64(a.into(), b.into()).simd_into(token)
7621 }
7622 );
7623 kernel(self, a, b)
7624 }
7625 #[inline(always)]
7626 fn any_true_mask64x2(self, a: mask64x2<Self>) -> bool {
7627 crate::kernel!(
7628 #[inline(always)]
7629 fn kernel(token: Avx2, a: mask64x2<Avx2>) -> bool {
7630 _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 != 0
7631 }
7632 );
7633 kernel(self, a)
7634 }
7635 #[inline(always)]
7636 fn all_true_mask64x2(self, a: mask64x2<Self>) -> bool {
7637 crate::kernel!(
7638 #[inline(always)]
7639 fn kernel(token: Avx2, a: mask64x2<Avx2>) -> bool {
7640 _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 == 0b11
7641 }
7642 );
7643 kernel(self, a)
7644 }
7645 #[inline(always)]
7646 fn any_false_mask64x2(self, a: mask64x2<Self>) -> bool {
7647 crate::kernel!(
7648 #[inline(always)]
7649 fn kernel(token: Avx2, a: mask64x2<Avx2>) -> bool {
7650 _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 != 0b11
7651 }
7652 );
7653 kernel(self, a)
7654 }
7655 #[inline(always)]
7656 fn all_false_mask64x2(self, a: mask64x2<Self>) -> bool {
7657 crate::kernel!(
7658 #[inline(always)]
7659 fn kernel(token: Avx2, a: mask64x2<Avx2>) -> bool {
7660 _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 == 0
7661 }
7662 );
7663 kernel(self, a)
7664 }
7665 #[inline(always)]
7666 fn combine_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x4<Self> {
7667 crate::kernel!(
7668 #[inline(always)]
7669 fn kernel(token: Avx2, a: mask64x2<Avx2>, b: mask64x2<Avx2>) -> mask64x4<Avx2> {
7670 _mm256_setr_m128i(a.into(), b.into()).simd_into(token)
7671 }
7672 );
7673 kernel(self, a, b)
7674 }
7675 #[inline(always)]
7676 fn narrow_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask32x4<Self> {
7677 crate::kernel!(
7678 #[inline(always)]
7679 fn kernel(token: Avx2, a: mask64x2<Avx2>, b: mask64x2<Avx2>) -> mask32x4<Avx2> {
7680 _mm_castps_si128(_mm_shuffle_ps::<0x88>(
7681 _mm_castsi128_ps(a.into()),
7682 _mm_castsi128_ps(b.into()),
7683 ))
7684 .simd_into(token)
7685 }
7686 );
7687 kernel(self, a, b)
7688 }
7689 #[inline(always)]
7690 fn abs_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
7691 crate::kernel!(
7692 #[inline(always)]
7693 fn kernel(token: Avx2, a: f32x8<Avx2>) -> f32x8<Avx2> {
7694 _mm256_andnot_ps(_mm256_set1_ps(-0.0), a.into()).simd_into(token)
7695 }
7696 );
7697 kernel(self, a)
7698 }
7699 #[inline(always)]
7700 fn splat_f32x8(self, val: f32) -> f32x8<Self> {
7701 crate::kernel!(
7702 #[inline(always)]
7703 fn kernel(token: Avx2, val: f32) -> f32x8<Avx2> {
7704 _mm256_set1_ps(val).simd_into(token)
7705 }
7706 );
7707 kernel(self, val)
7708 }
7709 #[inline(always)]
7710 fn reverse_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
7711 let indices: u8x32<Self> = [
7712 28, 29, 30, 31, 24, 25, 26, 27, 20, 21, 22, 23, 16, 17, 18, 19, 12, 13, 14, 15, 8, 9,
7713 10, 11, 4, 5, 6, 7, 0, 1, 2, 3,
7714 ]
7715 .simd_into(self);
7716 self.swizzle_dyn_f32x8(a, indices)
7717 }
7718 #[inline(always)]
7719 fn slide_f32x8<const SHIFT: usize>(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
7720 if SHIFT >= 8usize {
7721 return b;
7722 }
7723 let result = cross_block_alignr_256x1(
7724 self,
7725 Bytes::to_bytes(b).val.0,
7726 Bytes::to_bytes(a).val.0,
7727 SHIFT * 4usize,
7728 );
7729 Bytes::from_bytes(u8x32 {
7730 val: crate::support::Aligned256(result),
7731 simd: self,
7732 })
7733 }
7734 #[inline(always)]
7735 fn slide_within_blocks_f32x8<const SHIFT: usize>(
7736 self,
7737 a: f32x8<Self>,
7738 b: f32x8<Self>,
7739 ) -> f32x8<Self> {
7740 if SHIFT >= 4usize {
7741 return b;
7742 }
7743 let result = dyn_alignr_256(
7744 self,
7745 Bytes::to_bytes(b).val.0,
7746 Bytes::to_bytes(a).val.0,
7747 SHIFT * 4usize,
7748 );
7749 Bytes::from_bytes(u8x32 {
7750 val: crate::support::Aligned256(result),
7751 simd: self,
7752 })
7753 }
7754 #[inline(always)]
7755 fn compress_f32x8(self, values: f32x8<Self>, mask: mask32x8<Self>) -> f32x8<Self> {
7756 crate::kernel!(
7757 #[inline(always)]
7758 fn kernel(token: Avx2, values: f32x8<Avx2>, mask: mask32x8<Avx2>) -> f32x8<Avx2> {
7759 let bits = token.to_bitmask_mask32x8(mask) as usize & 255;
7760 let control: __m256i = crate::transmute::checked_transmute_copy(
7761 &crate::support::compact_256::COMPRESS_32[bits],
7762 );
7763 let reordered =
7764 _mm256_permutevar8x32_epi32(_mm256_castps_si256(values.into()), control);
7765 let result = _mm256_andnot_si256(_mm256_srai_epi32::<31>(control), reordered);
7766 _mm256_castsi256_ps(result).simd_into(token)
7767 }
7768 );
7769 kernel(self, values, mask)
7770 }
7771 #[inline(always)]
7772 fn compress_merge_f32x8(
7773 self,
7774 values: f32x8<Self>,
7775 mask: mask32x8<Self>,
7776 merge: f32x8<Self>,
7777 ) -> f32x8<Self> {
7778 crate::kernel!(
7779 #[inline(always)]
7780 fn kernel(
7781 token: Avx2,
7782 values: f32x8<Avx2>,
7783 mask: mask32x8<Avx2>,
7784 merge: f32x8<Avx2>,
7785 ) -> f32x8<Avx2> {
7786 let bits = token.to_bitmask_mask32x8(mask) as usize & 255;
7787 let control: __m256i = crate::transmute::checked_transmute_copy(
7788 &crate::support::compact_256::COMPRESS_32[bits],
7789 );
7790 let reordered =
7791 _mm256_permutevar8x32_epi32(_mm256_castps_si256(values.into()), control);
7792 let result =
7793 _mm256_blendv_epi8(reordered, _mm256_castps_si256(merge.into()), control);
7794 _mm256_castsi256_ps(result).simd_into(token)
7795 }
7796 );
7797 kernel(self, values, mask, merge)
7798 }
7799 #[inline(always)]
7800 fn expand_f32x8(self, values: f32x8<Self>, mask: mask32x8<Self>) -> f32x8<Self> {
7801 crate::kernel!(
7802 #[inline(always)]
7803 fn kernel(token: Avx2, values: f32x8<Avx2>, mask: mask32x8<Avx2>) -> f32x8<Avx2> {
7804 let bits = token.to_bitmask_mask32x8(mask) as usize & 255;
7805 let control: __m256i = crate::transmute::checked_transmute_copy(
7806 &crate::support::compact_256::EXPAND_32[bits],
7807 );
7808 let reordered =
7809 _mm256_permutevar8x32_epi32(_mm256_castps_si256(values.into()), control);
7810 let result = _mm256_and_si256(reordered, mask.into());
7811 _mm256_castsi256_ps(result).simd_into(token)
7812 }
7813 );
7814 kernel(self, values, mask)
7815 }
7816 #[inline(always)]
7817 fn expand_merge_f32x8(
7818 self,
7819 values: f32x8<Self>,
7820 mask: mask32x8<Self>,
7821 merge: f32x8<Self>,
7822 ) -> f32x8<Self> {
7823 crate::kernel!(
7824 #[inline(always)]
7825 fn kernel(
7826 token: Avx2,
7827 values: f32x8<Avx2>,
7828 mask: mask32x8<Avx2>,
7829 merge: f32x8<Avx2>,
7830 ) -> f32x8<Avx2> {
7831 let bits = token.to_bitmask_mask32x8(mask) as usize & 255;
7832 let control: __m256i = crate::transmute::checked_transmute_copy(
7833 &crate::support::compact_256::EXPAND_32[bits],
7834 );
7835 let reordered =
7836 _mm256_permutevar8x32_epi32(_mm256_castps_si256(values.into()), control);
7837 let result =
7838 _mm256_blendv_epi8(_mm256_castps_si256(merge.into()), reordered, mask.into());
7839 _mm256_castsi256_ps(result).simd_into(token)
7840 }
7841 );
7842 kernel(self, values, mask, merge)
7843 }
7844 #[inline(always)]
7845 fn neg_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
7846 crate::kernel!(
7847 #[inline(always)]
7848 fn kernel(token: Avx2, a: f32x8<Avx2>) -> f32x8<Avx2> {
7849 _mm256_xor_ps(a.into(), _mm256_set1_ps(-0.0)).simd_into(token)
7850 }
7851 );
7852 kernel(self, a)
7853 }
7854 #[inline(always)]
7855 fn sqrt_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
7856 crate::kernel!(
7857 #[inline(always)]
7858 fn kernel(token: Avx2, a: f32x8<Avx2>) -> f32x8<Avx2> {
7859 _mm256_sqrt_ps(a.into()).simd_into(token)
7860 }
7861 );
7862 kernel(self, a)
7863 }
7864 #[inline(always)]
7865 fn approximate_recip_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
7866 crate::kernel!(
7867 #[inline(always)]
7868 fn kernel(token: Avx2, a: f32x8<Avx2>) -> f32x8<Avx2> {
7869 _mm256_rcp_ps(a.into()).simd_into(token)
7870 }
7871 );
7872 kernel(self, a)
7873 }
7874 #[inline(always)]
7875 fn add_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
7876 crate::kernel!(
7877 #[inline(always)]
7878 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
7879 _mm256_add_ps(a.into(), b.into()).simd_into(token)
7880 }
7881 );
7882 kernel(self, a, b)
7883 }
7884 #[inline(always)]
7885 fn sub_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
7886 crate::kernel!(
7887 #[inline(always)]
7888 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
7889 _mm256_sub_ps(a.into(), b.into()).simd_into(token)
7890 }
7891 );
7892 kernel(self, a, b)
7893 }
7894 #[inline(always)]
7895 fn mul_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
7896 crate::kernel!(
7897 #[inline(always)]
7898 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
7899 _mm256_mul_ps(a.into(), b.into()).simd_into(token)
7900 }
7901 );
7902 kernel(self, a, b)
7903 }
7904 #[inline(always)]
7905 fn div_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
7906 crate::kernel!(
7907 #[inline(always)]
7908 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
7909 _mm256_div_ps(a.into(), b.into()).simd_into(token)
7910 }
7911 );
7912 kernel(self, a, b)
7913 }
7914 #[inline(always)]
7915 fn copysign_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
7916 crate::kernel!(
7917 #[inline(always)]
7918 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
7919 let mask = _mm256_set1_ps(-0.0);
7920 _mm256_or_ps(
7921 _mm256_and_ps(mask, b.into()),
7922 _mm256_andnot_ps(mask, a.into()),
7923 )
7924 .simd_into(token)
7925 }
7926 );
7927 kernel(self, a, b)
7928 }
7929 #[inline(always)]
7930 fn max_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
7931 crate::kernel!(
7932 #[inline(always)]
7933 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
7934 _mm256_max_ps(a.into(), b.into()).simd_into(token)
7935 }
7936 );
7937 kernel(self, a, b)
7938 }
7939 #[inline(always)]
7940 fn min_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
7941 crate::kernel!(
7942 #[inline(always)]
7943 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
7944 _mm256_min_ps(a.into(), b.into()).simd_into(token)
7945 }
7946 );
7947 kernel(self, a, b)
7948 }
7949 #[inline(always)]
7950 fn max_precise_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
7951 crate::kernel!(
7952 #[inline(always)]
7953 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
7954 let intermediate = _mm256_max_ps(a.into(), b.into());
7955 let b_is_nan = _mm256_cmp_ps::<3i32>(b.into(), b.into());
7956 _mm256_blendv_ps(intermediate, a.into(), b_is_nan).simd_into(token)
7957 }
7958 );
7959 kernel(self, a, b)
7960 }
7961 #[inline(always)]
7962 fn min_precise_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
7963 crate::kernel!(
7964 #[inline(always)]
7965 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
7966 let intermediate = _mm256_min_ps(a.into(), b.into());
7967 let b_is_nan = _mm256_cmp_ps::<3i32>(b.into(), b.into());
7968 _mm256_blendv_ps(intermediate, a.into(), b_is_nan).simd_into(token)
7969 }
7970 );
7971 kernel(self, a, b)
7972 }
7973 #[inline(always)]
7974 fn clamp_f32x8(self, a: f32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
7975 crate::kernel!(
7976 #[inline(always)]
7977 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>, c: f32x8<Avx2>) -> f32x8<Avx2> {
7978 debug_assert!(
7979 b.simd_le(c).all_true(),
7980 "clamp requires min <= max for all elements"
7981 );
7982 _mm256_min_ps(c.into(), _mm256_max_ps(b.into(), a.into())).simd_into(token)
7983 }
7984 );
7985 kernel(self, a, b, c)
7986 }
7987 #[inline(always)]
7988 fn simd_eq_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
7989 crate::kernel!(
7990 #[inline(always)]
7991 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> mask32x8<Avx2> {
7992 _mm256_castps_si256(_mm256_cmp_ps::<0i32>(a.into(), b.into())).simd_into(token)
7993 }
7994 );
7995 kernel(self, a, b)
7996 }
7997 #[inline(always)]
7998 fn simd_ne_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
7999 crate::kernel!(
8000 #[inline(always)]
8001 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> mask32x8<Avx2> {
8002 _mm256_castps_si256(_mm256_cmp_ps::<4i32>(a.into(), b.into())).simd_into(token)
8003 }
8004 );
8005 kernel(self, a, b)
8006 }
8007 #[inline(always)]
8008 fn simd_lt_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
8009 crate::kernel!(
8010 #[inline(always)]
8011 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> mask32x8<Avx2> {
8012 _mm256_castps_si256(_mm256_cmp_ps::<17i32>(a.into(), b.into())).simd_into(token)
8013 }
8014 );
8015 kernel(self, a, b)
8016 }
8017 #[inline(always)]
8018 fn simd_le_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
8019 crate::kernel!(
8020 #[inline(always)]
8021 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> mask32x8<Avx2> {
8022 _mm256_castps_si256(_mm256_cmp_ps::<18i32>(a.into(), b.into())).simd_into(token)
8023 }
8024 );
8025 kernel(self, a, b)
8026 }
8027 #[inline(always)]
8028 fn zip_low_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
8029 crate::kernel!(
8030 #[inline(always)]
8031 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
8032 let lo = _mm256_unpacklo_ps(a.into(), b.into());
8033 let hi = _mm256_unpackhi_ps(a.into(), b.into());
8034 _mm256_permute2f128_ps::<0b0010_0000>(lo, hi).simd_into(token)
8035 }
8036 );
8037 kernel(self, a, b)
8038 }
8039 #[inline(always)]
8040 fn zip_high_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
8041 crate::kernel!(
8042 #[inline(always)]
8043 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
8044 let lo = _mm256_unpacklo_ps(a.into(), b.into());
8045 let hi = _mm256_unpackhi_ps(a.into(), b.into());
8046 _mm256_permute2f128_ps::<0b0011_0001>(lo, hi).simd_into(token)
8047 }
8048 );
8049 kernel(self, a, b)
8050 }
8051 #[inline(always)]
8052 fn unzip_low_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
8053 crate::kernel!(
8054 #[inline(always)]
8055 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
8056 let t1 =
8057 _mm256_permutevar8x32_ps(a.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7));
8058 let t2 =
8059 _mm256_permutevar8x32_ps(b.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7));
8060 _mm256_permute2f128_ps::<0b0010_0000>(t1, t2).simd_into(token)
8061 }
8062 );
8063 kernel(self, a, b)
8064 }
8065 #[inline(always)]
8066 fn unzip_high_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
8067 crate::kernel!(
8068 #[inline(always)]
8069 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
8070 let t1 =
8071 _mm256_permutevar8x32_ps(a.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7));
8072 let t2 =
8073 _mm256_permutevar8x32_ps(b.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7));
8074 _mm256_permute2f128_ps::<0b0011_0001>(t1, t2).simd_into(token)
8075 }
8076 );
8077 kernel(self, a, b)
8078 }
8079 #[inline(always)]
8080 fn interleave_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> (f32x8<Self>, f32x8<Self>) {
8081 crate::kernel!(
8082 #[inline(always)]
8083 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> (f32x8<Avx2>, f32x8<Avx2>) {
8084 let lo = _mm256_unpacklo_ps(a.into(), b.into());
8085 let hi = _mm256_unpackhi_ps(a.into(), b.into());
8086 (
8087 _mm256_permute2f128_ps::<0b0010_0000>(lo, hi).simd_into(token),
8088 _mm256_permute2f128_ps::<0b0011_0001>(lo, hi).simd_into(token),
8089 )
8090 }
8091 );
8092 kernel(self, a, b)
8093 }
8094 #[inline(always)]
8095 fn deinterleave_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> (f32x8<Self>, f32x8<Self>) {
8096 crate::kernel!(
8097 #[inline(always)]
8098 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> (f32x8<Avx2>, f32x8<Avx2>) {
8099 let t1 =
8100 _mm256_permutevar8x32_ps(a.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7));
8101 let t2 =
8102 _mm256_permutevar8x32_ps(b.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7));
8103 (
8104 _mm256_permute2f128_ps::<0b0010_0000>(t1, t2).simd_into(token),
8105 _mm256_permute2f128_ps::<0b0011_0001>(t1, t2).simd_into(token),
8106 )
8107 }
8108 );
8109 kernel(self, a, b)
8110 }
8111 #[inline(always)]
8112 fn mul_add_f32x8(self, a: f32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
8113 crate::kernel!(
8114 #[inline(always)]
8115 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>, c: f32x8<Avx2>) -> f32x8<Avx2> {
8116 _mm256_fmadd_ps(a.into(), b.into(), c.into()).simd_into(token)
8117 }
8118 );
8119 kernel(self, a, b, c)
8120 }
8121 #[inline(always)]
8122 fn mul_add_precise_f32x8(self, a: f32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
8123 self.mul_add_f32x8(a, b, c)
8124 }
8125 #[inline(always)]
8126 fn mul_sub_f32x8(self, a: f32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
8127 crate::kernel!(
8128 #[inline(always)]
8129 fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>, c: f32x8<Avx2>) -> f32x8<Avx2> {
8130 _mm256_fmsub_ps(a.into(), b.into(), c.into()).simd_into(token)
8131 }
8132 );
8133 kernel(self, a, b, c)
8134 }
8135 #[inline(always)]
8136 fn mul_sub_precise_f32x8(self, a: f32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
8137 self.mul_sub_f32x8(a, b, c)
8138 }
8139 #[inline(always)]
8140 fn floor_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
8141 crate::kernel!(
8142 #[inline(always)]
8143 fn kernel(token: Avx2, a: f32x8<Avx2>) -> f32x8<Avx2> {
8144 _mm256_round_ps::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
8145 .simd_into(token)
8146 }
8147 );
8148 kernel(self, a)
8149 }
8150 #[inline(always)]
8151 fn ceil_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
8152 crate::kernel!(
8153 #[inline(always)]
8154 fn kernel(token: Avx2, a: f32x8<Avx2>) -> f32x8<Avx2> {
8155 _mm256_round_ps::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
8156 .simd_into(token)
8157 }
8158 );
8159 kernel(self, a)
8160 }
8161 #[inline(always)]
8162 fn round_ties_even_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
8163 crate::kernel!(
8164 #[inline(always)]
8165 fn kernel(token: Avx2, a: f32x8<Avx2>) -> f32x8<Avx2> {
8166 _mm256_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
8167 .simd_into(token)
8168 }
8169 );
8170 kernel(self, a)
8171 }
8172 #[inline(always)]
8173 fn round_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
8174 let bias = self.splat_f32x8(const { f32::next_down(0.5) });
8175 (a + self.copysign_f32x8(bias, a)).trunc()
8176 }
8177 #[inline(always)]
8178 fn fract_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
8179 a - self.trunc_f32x8(a)
8180 }
8181 #[inline(always)]
8182 fn trunc_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
8183 crate::kernel!(
8184 #[inline(always)]
8185 fn kernel(token: Avx2, a: f32x8<Avx2>) -> f32x8<Avx2> {
8186 _mm256_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
8187 .simd_into(token)
8188 }
8189 );
8190 kernel(self, a)
8191 }
8192 #[inline(always)]
8193 fn to_degrees_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
8194 a * const { f32::to_degrees(1.0) }
8195 }
8196 #[inline(always)]
8197 fn to_radians_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
8198 a * const { f32::to_radians(1.0) }
8199 }
8200 #[inline(always)]
8201 fn is_nan_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
8202 a.simd_ne(a)
8203 }
8204 #[inline(always)]
8205 fn is_infinite_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
8206 a.abs().simd_eq(f32::INFINITY)
8207 }
8208 #[inline(always)]
8209 fn is_finite_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
8210 a.abs().simd_lt(f32::INFINITY)
8211 }
8212 #[inline(always)]
8213 fn is_subnormal_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
8214 let i: i32x8<Self> = a.bitcast();
8215 (i & 0x7F80_0000_i32).simd_eq(0) & (i & 0x007F_FFFF_i32).simd_ne(0)
8216 }
8217 #[inline(always)]
8218 fn is_normal_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
8219 let i: i32x8<Self> = a.bitcast();
8220 let exp = i & 0x7F80_0000_i32;
8221 exp.simd_ne(0) & exp.simd_ne(0x7F80_0000_i32)
8222 }
8223 #[inline(always)]
8224 fn is_sign_positive_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
8225 let i: i32x8<Self> = a.bitcast();
8226 (i & i32::MIN).simd_eq(0)
8227 }
8228 #[inline(always)]
8229 fn is_sign_negative_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
8230 !a.is_sign_positive()
8231 }
8232 #[inline(always)]
8233 fn select_f32x8(self, a: mask32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
8234 crate::kernel!(
8235 #[inline(always)]
8236 fn kernel(
8237 token: Avx2,
8238 a: mask32x8<Avx2>,
8239 b: f32x8<Avx2>,
8240 c: f32x8<Avx2>,
8241 ) -> f32x8<Avx2> {
8242 _mm256_blendv_ps(c.into(), b.into(), _mm256_castsi256_ps(a.into())).simd_into(token)
8243 }
8244 );
8245 kernel(self, a, b, c)
8246 }
8247 #[inline(always)]
8248 fn combine_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x16<Self> {
8249 f32x16 {
8250 val: crate::support::Aligned512([a.val.0, b.val.0]),
8251 simd: self,
8252 }
8253 }
8254 #[inline(always)]
8255 fn split_f32x8(self, a: f32x8<Self>) -> (f32x4<Self>, f32x4<Self>) {
8256 crate::kernel!(
8257 #[inline(always)]
8258 fn kernel(token: Avx2, a: f32x8<Avx2>) -> (f32x4<Avx2>, f32x4<Avx2>) {
8259 (
8260 _mm256_extractf128_ps::<0>(a.into()).simd_into(token),
8261 _mm256_extractf128_ps::<1>(a.into()).simd_into(token),
8262 )
8263 }
8264 );
8265 kernel(self, a)
8266 }
8267 #[inline(always)]
8268 fn widen_f32x8(self, a: f32x8<Self>) -> (f64x4<Self>, f64x4<Self>) {
8269 crate::kernel!(
8270 #[inline(always)]
8271 fn kernel(token: Avx2, a: f32x8<Avx2>) -> (f64x4<Avx2>, f64x4<Avx2>) {
8272 let raw = a.into();
8273 (
8274 _mm256_cvtps_pd(_mm256_castps256_ps128(raw)).simd_into(token),
8275 _mm256_cvtps_pd(_mm256_extractf128_ps::<1>(raw)).simd_into(token),
8276 )
8277 }
8278 );
8279 kernel(self, a)
8280 }
8281 #[inline(always)]
8282 fn cvt_u32_f32x8(self, a: f32x8<Self>) -> u32x8<Self> {
8283 crate::kernel!(
8284 #[inline(always)]
8285 fn kernel(token: Avx2, a: f32x8<Avx2>) -> u32x8<Avx2> {
8286 let mut converted = _mm256_cvttps_epi32(a.into());
8287 let in_range = _mm256_cmp_ps::<17i32>(a.into(), _mm256_set1_ps(2147483648.0));
8288 let all_in_range = _mm256_movemask_ps(in_range) == 0b11111111;
8289 if !all_in_range {
8290 let excess = _mm256_sub_ps(a.into(), _mm256_set1_ps(2147483648.0));
8291 let excess_converted = _mm256_cvttps_epi32(_mm256_andnot_ps(in_range, excess));
8292 converted = _mm256_add_epi32(converted, excess_converted);
8293 }
8294 converted.simd_into(token)
8295 }
8296 );
8297 kernel(self, a)
8298 }
8299 #[inline(always)]
8300 fn cvt_u32_precise_f32x8(self, a: f32x8<Self>) -> u32x8<Self> {
8301 crate::kernel!(
8302 #[inline(always)]
8303 fn kernel(token: Avx2, a: f32x8<Avx2>) -> u32x8<Avx2> {
8304 let a = _mm256_max_ps(a.into(), _mm256_setzero_ps());
8305 let mut converted = _mm256_cvttps_epi32(a);
8306 let in_range = _mm256_cmp_ps::<17i32>(a, _mm256_set1_ps(2147483648.0));
8307 let all_in_range = _mm256_movemask_ps(in_range) == 0b11111111;
8308 if !all_in_range {
8309 let exceeds_unsigned_range = _mm256_castps_si256(_mm256_cmp_ps::<17i32>(
8310 _mm256_set1_ps(4294967040.0),
8311 a,
8312 ));
8313 let excess = _mm256_sub_ps(a, _mm256_set1_ps(2147483648.0));
8314 let excess_converted = _mm256_cvttps_epi32(_mm256_andnot_ps(in_range, excess));
8315 converted = _mm256_add_epi32(converted, excess_converted);
8316 converted = _mm256_or_si256(converted, exceeds_unsigned_range);
8317 }
8318 converted.simd_into(token)
8319 }
8320 );
8321 kernel(self, a)
8322 }
8323 #[inline(always)]
8324 fn cvt_i32_f32x8(self, a: f32x8<Self>) -> i32x8<Self> {
8325 crate::kernel!(
8326 #[inline(always)]
8327 fn kernel(token: Avx2, a: f32x8<Avx2>) -> i32x8<Avx2> {
8328 _mm256_cvttps_epi32(a.into()).simd_into(token)
8329 }
8330 );
8331 kernel(self, a)
8332 }
8333 #[inline(always)]
8334 fn cvt_i32_precise_f32x8(self, a: f32x8<Self>) -> i32x8<Self> {
8335 crate::kernel!(
8336 #[inline(always)]
8337 fn kernel(token: Avx2, a: f32x8<Avx2>) -> i32x8<Avx2> {
8338 let a = a.into();
8339 let converted = _mm256_cvttps_epi32(a);
8340 let positive_overflow =
8341 _mm256_castps_si256(_mm256_cmp_ps::<18i32>(_mm256_set1_ps(2147483648.0), a));
8342 let converted = _mm256_xor_si256(converted, positive_overflow);
8343 let is_not_nan = _mm256_castps_si256(_mm256_cmp_ps::<7i32>(a, a));
8344 _mm256_and_si256(converted, is_not_nan).simd_into(token)
8345 }
8346 );
8347 kernel(self, a)
8348 }
8349 #[inline(always)]
8350 fn abs_i8x32(self, a: i8x32<Self>) -> i8x32<Self> {
8351 crate::kernel!(
8352 #[inline(always)]
8353 fn kernel(token: Avx2, a: i8x32<Avx2>) -> i8x32<Avx2> {
8354 _mm256_abs_epi8(a.into()).simd_into(token)
8355 }
8356 );
8357 kernel(self, a)
8358 }
8359 #[inline(always)]
8360 fn splat_i8x32(self, val: i8) -> i8x32<Self> {
8361 crate::kernel!(
8362 #[inline(always)]
8363 fn kernel(token: Avx2, val: i8) -> i8x32<Avx2> {
8364 _mm256_set1_epi8(val).simd_into(token)
8365 }
8366 );
8367 kernel(self, val)
8368 }
8369 #[inline(always)]
8370 fn reverse_i8x32(self, a: i8x32<Self>) -> i8x32<Self> {
8371 let indices: u8x32<Self> = [
8372 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10,
8373 9, 8, 7, 6, 5, 4, 3, 2, 1, 0,
8374 ]
8375 .simd_into(self);
8376 self.swizzle_dyn_i8x32(a, indices)
8377 }
8378 #[inline(always)]
8379 fn slide_i8x32<const SHIFT: usize>(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8380 if SHIFT >= 32usize {
8381 return b;
8382 }
8383 let result = cross_block_alignr_256x1(
8384 self,
8385 Bytes::to_bytes(b).val.0,
8386 Bytes::to_bytes(a).val.0,
8387 SHIFT,
8388 );
8389 Bytes::from_bytes(u8x32 {
8390 val: crate::support::Aligned256(result),
8391 simd: self,
8392 })
8393 }
8394 #[inline(always)]
8395 fn slide_within_blocks_i8x32<const SHIFT: usize>(
8396 self,
8397 a: i8x32<Self>,
8398 b: i8x32<Self>,
8399 ) -> i8x32<Self> {
8400 if SHIFT >= 16usize {
8401 return b;
8402 }
8403 let result = dyn_alignr_256(
8404 self,
8405 Bytes::to_bytes(b).val.0,
8406 Bytes::to_bytes(a).val.0,
8407 SHIFT,
8408 );
8409 Bytes::from_bytes(u8x32 {
8410 val: crate::support::Aligned256(result),
8411 simd: self,
8412 })
8413 }
8414 #[inline(always)]
8415 fn compress_i8x32(self, values: i8x32<Self>, mask: mask8x32<Self>) -> i8x32<Self> {
8416 let mask = mask8x32 {
8417 val: crate::transmute::checked_transmute_copy(&mask.val),
8418 simd: self,
8419 };
8420 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
8421 }
8422 #[inline(always)]
8423 fn compress_merge_i8x32(
8424 self,
8425 values: i8x32<Self>,
8426 mask: mask8x32<Self>,
8427 merge: i8x32<Self>,
8428 ) -> i8x32<Self> {
8429 let mask = mask8x32 {
8430 val: crate::transmute::checked_transmute_copy(&mask.val),
8431 simd: self,
8432 };
8433 Bytes::from_bytes(self.compress_merge_u8x32(
8434 Bytes::to_bytes(values),
8435 mask,
8436 Bytes::to_bytes(merge),
8437 ))
8438 }
8439 #[inline(always)]
8440 fn expand_i8x32(self, values: i8x32<Self>, mask: mask8x32<Self>) -> i8x32<Self> {
8441 let mask = mask8x32 {
8442 val: crate::transmute::checked_transmute_copy(&mask.val),
8443 simd: self,
8444 };
8445 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
8446 }
8447 #[inline(always)]
8448 fn expand_merge_i8x32(
8449 self,
8450 values: i8x32<Self>,
8451 mask: mask8x32<Self>,
8452 merge: i8x32<Self>,
8453 ) -> i8x32<Self> {
8454 let mask = mask8x32 {
8455 val: crate::transmute::checked_transmute_copy(&mask.val),
8456 simd: self,
8457 };
8458 Bytes::from_bytes(self.expand_merge_u8x32(
8459 Bytes::to_bytes(values),
8460 mask,
8461 Bytes::to_bytes(merge),
8462 ))
8463 }
8464 #[inline(always)]
8465 fn count_ones_i8x32(self, a: i8x32<Self>) -> i8x32<Self> {
8466 crate::kernel!(
8467 #[inline(always)]
8468 fn kernel(token: Avx2, a: i8x32<Avx2>) -> i8x32<Avx2> {
8469 let value = a.into();
8470 let nibble_mask = _mm256_set1_epi8(0x0f);
8471 let lookup = _mm256_setr_epi8(
8472 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 0, 1, 1, 2, 1, 2, 2, 3, 1, 2,
8473 2, 3, 2, 3, 3, 4,
8474 );
8475 let low = _mm256_and_si256(value, nibble_mask);
8476 let high = _mm256_and_si256(_mm256_srli_epi16::<4>(value), nibble_mask);
8477 let byte_counts = _mm256_add_epi8(
8478 _mm256_shuffle_epi8(lookup, low),
8479 _mm256_shuffle_epi8(lookup, high),
8480 );
8481 byte_counts.simd_into(token)
8482 }
8483 );
8484 kernel(self, a)
8485 }
8486 #[inline(always)]
8487 fn count_zeros_i8x32(self, a: i8x32<Self>) -> i8x32<Self> {
8488 self.count_ones_i8x32(self.not_i8x32(a))
8489 }
8490 #[inline(always)]
8491 fn add_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8492 crate::kernel!(
8493 #[inline(always)]
8494 fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
8495 _mm256_add_epi8(a.into(), b.into()).simd_into(token)
8496 }
8497 );
8498 kernel(self, a, b)
8499 }
8500 #[inline(always)]
8501 fn saturating_add_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8502 crate::kernel!(
8503 #[inline(always)]
8504 fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
8505 _mm256_adds_epi8(a.into(), b.into()).simd_into(token)
8506 }
8507 );
8508 kernel(self, a, b)
8509 }
8510 #[inline(always)]
8511 fn sub_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8512 crate::kernel!(
8513 #[inline(always)]
8514 fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
8515 _mm256_sub_epi8(a.into(), b.into()).simd_into(token)
8516 }
8517 );
8518 kernel(self, a, b)
8519 }
8520 #[inline(always)]
8521 fn saturating_sub_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8522 crate::kernel!(
8523 #[inline(always)]
8524 fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
8525 _mm256_subs_epi8(a.into(), b.into()).simd_into(token)
8526 }
8527 );
8528 kernel(self, a, b)
8529 }
8530 #[inline(always)]
8531 fn mul_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8532 crate::kernel!(
8533 #[inline(always)]
8534 fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
8535 let a = a.into();
8536 let b = b.into();
8537 let low_mask = _mm256_set1_epi16(0xFF);
8538 let dst_even = _mm256_mullo_epi16(a, b);
8539 let dst_odd = _mm256_maddubs_epi16(a, _mm256_andnot_si256(low_mask, b));
8540 _mm256_or_si256(
8541 _mm256_slli_epi16(dst_odd, 8),
8542 _mm256_and_si256(dst_even, low_mask),
8543 )
8544 .simd_into(token)
8545 }
8546 );
8547 kernel(self, a, b)
8548 }
8549 #[inline(always)]
8550 fn and_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8551 crate::kernel!(
8552 #[inline(always)]
8553 fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
8554 _mm256_and_si256(a.into(), b.into()).simd_into(token)
8555 }
8556 );
8557 kernel(self, a, b)
8558 }
8559 #[inline(always)]
8560 fn or_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8561 crate::kernel!(
8562 #[inline(always)]
8563 fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
8564 _mm256_or_si256(a.into(), b.into()).simd_into(token)
8565 }
8566 );
8567 kernel(self, a, b)
8568 }
8569 #[inline(always)]
8570 fn xor_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8571 crate::kernel!(
8572 #[inline(always)]
8573 fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
8574 _mm256_xor_si256(a.into(), b.into()).simd_into(token)
8575 }
8576 );
8577 kernel(self, a, b)
8578 }
8579 #[inline(always)]
8580 fn not_i8x32(self, a: i8x32<Self>) -> i8x32<Self> {
8581 a ^ !0
8582 }
8583 #[inline(always)]
8584 fn shl_i8x32(self, a: i8x32<Self>, shift: u32) -> i8x32<Self> {
8585 crate::kernel!(
8586 #[inline(always)]
8587 fn kernel(token: Avx2, a: i8x32<Avx2>, shift: u32) -> i8x32<Avx2> {
8588 let val = a.into();
8589 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
8590 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
8591 let byte_mask = _mm256_set1_epi8(mask_byte);
8592 _mm256_sll_epi16(_mm256_and_si256(val, byte_mask), shift_count).simd_into(token)
8593 }
8594 );
8595 kernel(self, a, shift)
8596 }
8597 #[inline(always)]
8598 fn shlv_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8599 [
8600 i8::wrapping_shl(a[0usize], b[0usize] as u32),
8601 i8::wrapping_shl(a[1usize], b[1usize] as u32),
8602 i8::wrapping_shl(a[2usize], b[2usize] as u32),
8603 i8::wrapping_shl(a[3usize], b[3usize] as u32),
8604 i8::wrapping_shl(a[4usize], b[4usize] as u32),
8605 i8::wrapping_shl(a[5usize], b[5usize] as u32),
8606 i8::wrapping_shl(a[6usize], b[6usize] as u32),
8607 i8::wrapping_shl(a[7usize], b[7usize] as u32),
8608 i8::wrapping_shl(a[8usize], b[8usize] as u32),
8609 i8::wrapping_shl(a[9usize], b[9usize] as u32),
8610 i8::wrapping_shl(a[10usize], b[10usize] as u32),
8611 i8::wrapping_shl(a[11usize], b[11usize] as u32),
8612 i8::wrapping_shl(a[12usize], b[12usize] as u32),
8613 i8::wrapping_shl(a[13usize], b[13usize] as u32),
8614 i8::wrapping_shl(a[14usize], b[14usize] as u32),
8615 i8::wrapping_shl(a[15usize], b[15usize] as u32),
8616 i8::wrapping_shl(a[16usize], b[16usize] as u32),
8617 i8::wrapping_shl(a[17usize], b[17usize] as u32),
8618 i8::wrapping_shl(a[18usize], b[18usize] as u32),
8619 i8::wrapping_shl(a[19usize], b[19usize] as u32),
8620 i8::wrapping_shl(a[20usize], b[20usize] as u32),
8621 i8::wrapping_shl(a[21usize], b[21usize] as u32),
8622 i8::wrapping_shl(a[22usize], b[22usize] as u32),
8623 i8::wrapping_shl(a[23usize], b[23usize] as u32),
8624 i8::wrapping_shl(a[24usize], b[24usize] as u32),
8625 i8::wrapping_shl(a[25usize], b[25usize] as u32),
8626 i8::wrapping_shl(a[26usize], b[26usize] as u32),
8627 i8::wrapping_shl(a[27usize], b[27usize] as u32),
8628 i8::wrapping_shl(a[28usize], b[28usize] as u32),
8629 i8::wrapping_shl(a[29usize], b[29usize] as u32),
8630 i8::wrapping_shl(a[30usize], b[30usize] as u32),
8631 i8::wrapping_shl(a[31usize], b[31usize] as u32),
8632 ]
8633 .simd_into(self)
8634 }
8635 #[inline(always)]
8636 fn shr_i8x32(self, a: i8x32<Self>, shift: u32) -> i8x32<Self> {
8637 crate::kernel!(
8638 #[inline(always)]
8639 fn kernel(token: Avx2, a: i8x32<Avx2>, shift: u32) -> i8x32<Avx2> {
8640 let val = a.into();
8641 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
8642 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
8643 let byte_mask = _mm256_set1_epi8(mask_byte);
8644 let shifted = _mm256_srl_epi16(val, shift_count);
8645 let result = {
8646 let sign = _mm256_cmpgt_epi8(_mm256_setzero_si256(), val);
8647 _mm256_or_si256(
8648 _mm256_and_si256(shifted, byte_mask),
8649 _mm256_andnot_si256(byte_mask, sign),
8650 )
8651 };
8652 result.simd_into(token)
8653 }
8654 );
8655 kernel(self, a, shift)
8656 }
8657 #[inline(always)]
8658 fn shrv_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8659 [
8660 i8::wrapping_shr(a[0usize], b[0usize] as u32),
8661 i8::wrapping_shr(a[1usize], b[1usize] as u32),
8662 i8::wrapping_shr(a[2usize], b[2usize] as u32),
8663 i8::wrapping_shr(a[3usize], b[3usize] as u32),
8664 i8::wrapping_shr(a[4usize], b[4usize] as u32),
8665 i8::wrapping_shr(a[5usize], b[5usize] as u32),
8666 i8::wrapping_shr(a[6usize], b[6usize] as u32),
8667 i8::wrapping_shr(a[7usize], b[7usize] as u32),
8668 i8::wrapping_shr(a[8usize], b[8usize] as u32),
8669 i8::wrapping_shr(a[9usize], b[9usize] as u32),
8670 i8::wrapping_shr(a[10usize], b[10usize] as u32),
8671 i8::wrapping_shr(a[11usize], b[11usize] as u32),
8672 i8::wrapping_shr(a[12usize], b[12usize] as u32),
8673 i8::wrapping_shr(a[13usize], b[13usize] as u32),
8674 i8::wrapping_shr(a[14usize], b[14usize] as u32),
8675 i8::wrapping_shr(a[15usize], b[15usize] as u32),
8676 i8::wrapping_shr(a[16usize], b[16usize] as u32),
8677 i8::wrapping_shr(a[17usize], b[17usize] as u32),
8678 i8::wrapping_shr(a[18usize], b[18usize] as u32),
8679 i8::wrapping_shr(a[19usize], b[19usize] as u32),
8680 i8::wrapping_shr(a[20usize], b[20usize] as u32),
8681 i8::wrapping_shr(a[21usize], b[21usize] as u32),
8682 i8::wrapping_shr(a[22usize], b[22usize] as u32),
8683 i8::wrapping_shr(a[23usize], b[23usize] as u32),
8684 i8::wrapping_shr(a[24usize], b[24usize] as u32),
8685 i8::wrapping_shr(a[25usize], b[25usize] as u32),
8686 i8::wrapping_shr(a[26usize], b[26usize] as u32),
8687 i8::wrapping_shr(a[27usize], b[27usize] as u32),
8688 i8::wrapping_shr(a[28usize], b[28usize] as u32),
8689 i8::wrapping_shr(a[29usize], b[29usize] as u32),
8690 i8::wrapping_shr(a[30usize], b[30usize] as u32),
8691 i8::wrapping_shr(a[31usize], b[31usize] as u32),
8692 ]
8693 .simd_into(self)
8694 }
8695 #[inline(always)]
8696 fn max_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8697 crate::kernel!(
8698 #[inline(always)]
8699 fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
8700 _mm256_max_epi8(a.into(), b.into()).simd_into(token)
8701 }
8702 );
8703 kernel(self, a, b)
8704 }
8705 #[inline(always)]
8706 fn min_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8707 crate::kernel!(
8708 #[inline(always)]
8709 fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
8710 _mm256_min_epi8(a.into(), b.into()).simd_into(token)
8711 }
8712 );
8713 kernel(self, a, b)
8714 }
8715 #[inline(always)]
8716 fn clamp_i8x32(self, a: i8x32<Self>, b: i8x32<Self>, c: i8x32<Self>) -> i8x32<Self> {
8717 debug_assert!(
8718 b.simd_le(c).all_true(),
8719 "clamp requires min <= max for all elements"
8720 );
8721 a.max(b).min(c)
8722 }
8723 #[inline(always)]
8724 fn simd_eq_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> mask8x32<Self> {
8725 crate::kernel!(
8726 #[inline(always)]
8727 fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> mask8x32<Avx2> {
8728 _mm256_cmpeq_epi8(a.into(), b.into()).simd_into(token)
8729 }
8730 );
8731 kernel(self, a, b)
8732 }
8733 #[inline(always)]
8734 fn simd_lt_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> mask8x32<Self> {
8735 crate::kernel!(
8736 #[inline(always)]
8737 fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> mask8x32<Avx2> {
8738 _mm256_cmpgt_epi8(b.into(), a.into()).simd_into(token)
8739 }
8740 );
8741 kernel(self, a, b)
8742 }
8743 #[inline(always)]
8744 fn simd_le_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> mask8x32<Self> {
8745 crate::kernel!(
8746 #[inline(always)]
8747 fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> mask8x32<Avx2> {
8748 _mm256_cmpeq_epi8(_mm256_min_epi8(a.into(), b.into()), a.into()).simd_into(token)
8749 }
8750 );
8751 kernel(self, a, b)
8752 }
8753 #[inline(always)]
8754 fn zip_low_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8755 crate::kernel!(
8756 #[inline(always)]
8757 fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
8758 let lo = _mm256_unpacklo_epi8(a.into(), b.into());
8759 let hi = _mm256_unpackhi_epi8(a.into(), b.into());
8760 _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token)
8761 }
8762 );
8763 kernel(self, a, b)
8764 }
8765 #[inline(always)]
8766 fn zip_high_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8767 crate::kernel!(
8768 #[inline(always)]
8769 fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
8770 let lo = _mm256_unpacklo_epi8(a.into(), b.into());
8771 let hi = _mm256_unpackhi_epi8(a.into(), b.into());
8772 _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token)
8773 }
8774 );
8775 kernel(self, a, b)
8776 }
8777 #[inline(always)]
8778 fn unzip_low_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8779 crate::kernel!(
8780 #[inline(always)]
8781 fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
8782 let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
8783 a.into(),
8784 _mm256_setr_epi8(
8785 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10,
8786 12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
8787 ),
8788 ));
8789 let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
8790 b.into(),
8791 _mm256_setr_epi8(
8792 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10,
8793 12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
8794 ),
8795 ));
8796 _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token)
8797 }
8798 );
8799 kernel(self, a, b)
8800 }
8801 #[inline(always)]
8802 fn unzip_high_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8803 crate::kernel!(
8804 #[inline(always)]
8805 fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
8806 let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
8807 a.into(),
8808 _mm256_setr_epi8(
8809 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10,
8810 12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
8811 ),
8812 ));
8813 let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
8814 b.into(),
8815 _mm256_setr_epi8(
8816 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10,
8817 12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
8818 ),
8819 ));
8820 _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token)
8821 }
8822 );
8823 kernel(self, a, b)
8824 }
8825 #[inline(always)]
8826 fn interleave_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> (i8x32<Self>, i8x32<Self>) {
8827 crate::kernel!(
8828 #[inline(always)]
8829 fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> (i8x32<Avx2>, i8x32<Avx2>) {
8830 let lo = _mm256_unpacklo_epi8(a.into(), b.into());
8831 let hi = _mm256_unpackhi_epi8(a.into(), b.into());
8832 (
8833 _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token),
8834 _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token),
8835 )
8836 }
8837 );
8838 kernel(self, a, b)
8839 }
8840 #[inline(always)]
8841 fn deinterleave_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> (i8x32<Self>, i8x32<Self>) {
8842 crate::kernel!(
8843 #[inline(always)]
8844 fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> (i8x32<Avx2>, i8x32<Avx2>) {
8845 let mask = _mm256_setr_epi8(
8846 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, 12,
8847 14, 1, 3, 5, 7, 9, 11, 13, 15,
8848 );
8849 let a = _mm256_shuffle_epi8(a.into(), mask);
8850 let b = _mm256_shuffle_epi8(b.into(), mask);
8851 let low = _mm256_permute2x128_si256::<0b0010_0000>(a, b);
8852 let high = _mm256_permute2x128_si256::<0b0011_0001>(a, b);
8853 (
8854 _mm256_unpacklo_epi64(low, high).simd_into(token),
8855 _mm256_unpackhi_epi64(low, high).simd_into(token),
8856 )
8857 }
8858 );
8859 kernel(self, a, b)
8860 }
8861 #[inline(always)]
8862 fn select_i8x32(self, a: mask8x32<Self>, b: i8x32<Self>, c: i8x32<Self>) -> i8x32<Self> {
8863 crate::kernel!(
8864 #[inline(always)]
8865 fn kernel(
8866 token: Avx2,
8867 a: mask8x32<Avx2>,
8868 b: i8x32<Avx2>,
8869 c: i8x32<Avx2>,
8870 ) -> i8x32<Avx2> {
8871 _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
8872 }
8873 );
8874 kernel(self, a, b, c)
8875 }
8876 #[inline(always)]
8877 fn combine_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x64<Self> {
8878 i8x64 {
8879 val: crate::support::Aligned512([a.val.0, b.val.0]),
8880 simd: self,
8881 }
8882 }
8883 #[inline(always)]
8884 fn split_i8x32(self, a: i8x32<Self>) -> (i8x16<Self>, i8x16<Self>) {
8885 crate::kernel!(
8886 #[inline(always)]
8887 fn kernel(token: Avx2, a: i8x32<Avx2>) -> (i8x16<Avx2>, i8x16<Avx2>) {
8888 (
8889 _mm256_extracti128_si256::<0>(a.into()).simd_into(token),
8890 _mm256_extracti128_si256::<1>(a.into()).simd_into(token),
8891 )
8892 }
8893 );
8894 kernel(self, a)
8895 }
8896 #[inline(always)]
8897 fn neg_i8x32(self, a: i8x32<Self>) -> i8x32<Self> {
8898 crate::kernel!(
8899 #[inline(always)]
8900 fn kernel(token: Avx2, a: i8x32<Avx2>) -> i8x32<Avx2> {
8901 _mm256_sub_epi8(_mm256_setzero_si256(), a.into()).simd_into(token)
8902 }
8903 );
8904 kernel(self, a)
8905 }
8906 #[inline(always)]
8907 fn widen_i8x32(self, a: i8x32<Self>) -> (i16x16<Self>, i16x16<Self>) {
8908 crate::kernel!(
8909 #[inline(always)]
8910 fn kernel(token: Avx2, a: i8x32<Avx2>) -> (i16x16<Avx2>, i16x16<Avx2>) {
8911 let raw = a.into();
8912 (
8913 _mm256_cvtepi8_epi16(_mm256_castsi256_si128(raw)).simd_into(token),
8914 _mm256_cvtepi8_epi16(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
8915 )
8916 }
8917 );
8918 kernel(self, a)
8919 }
8920 #[inline(always)]
8921 fn widening_mul_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> (i16x16<Self>, i16x16<Self>) {
8922 let (a0, a1) = self.widen_i8x32(a);
8923 let (b0, b1) = self.widen_i8x32(b);
8924 (self.mul_i16x16(a0, b0), self.mul_i16x16(a1, b1))
8925 }
8926 #[inline(always)]
8927 fn widening_add_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> (i16x16<Self>, i16x16<Self>) {
8928 crate::kernel!(
8929 #[inline(always)]
8930 fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> (i16x16<Avx2>, i16x16<Avx2>) {
8931 let a = a.into();
8932 let b = b.into();
8933 let low = a;
8934 let high = b;
8935 let result_low = _mm256_unpacklo_epi8(low, high);
8936 let result_high = _mm256_unpackhi_epi8(low, high);
8937 let ones = _mm256_set1_epi8(1);
8938 let result_low = _mm256_maddubs_epi16(ones, result_low);
8939 let result_high = _mm256_maddubs_epi16(ones, result_high);
8940 let lo = _mm256_permute2x128_si256::<0x20>(result_low, result_high);
8941 let hi = _mm256_permute2x128_si256::<0x31>(result_low, result_high);
8942 let result_low = lo;
8943 let result_high = hi;
8944 (result_low.simd_into(token), result_high.simd_into(token))
8945 }
8946 );
8947 kernel(self, a, b)
8948 }
8949 #[inline(always)]
8950 fn pairwise_widening_add_i8x32(self, a: i8x32<Self>) -> i16x16<Self> {
8951 crate::kernel!(
8952 #[inline(always)]
8953 fn kernel(token: Avx2, a: i8x32<Avx2>) -> i16x16<Avx2> {
8954 ({ _mm256_maddubs_epi16(_mm256_set1_epi8(1), a.into()) }).simd_into(token)
8955 }
8956 );
8957 kernel(self, a)
8958 }
8959 #[inline(always)]
8960 fn pairwise_dot_product_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i16x16<Self> {
8961 let (low, high) = self.widening_mul_i8x32(a, b);
8962 let (even, odd) = self.deinterleave_i16x16(low, high);
8963 self.add_i16x16(even, odd)
8964 }
8965 #[inline(always)]
8966 fn splat_u8x32(self, val: u8) -> u8x32<Self> {
8967 crate::kernel!(
8968 #[inline(always)]
8969 fn kernel(token: Avx2, val: u8) -> u8x32<Avx2> {
8970 _mm256_set1_epi8(val.cast_signed()).simd_into(token)
8971 }
8972 );
8973 kernel(self, val)
8974 }
8975 #[inline(always)]
8976 fn reverse_u8x32(self, a: u8x32<Self>) -> u8x32<Self> {
8977 let indices: u8x32<Self> = [
8978 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10,
8979 9, 8, 7, 6, 5, 4, 3, 2, 1, 0,
8980 ]
8981 .simd_into(self);
8982 self.swizzle_dyn_u8x32(a, indices)
8983 }
8984 #[inline(always)]
8985 fn slide_u8x32<const SHIFT: usize>(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
8986 if SHIFT >= 32usize {
8987 return b;
8988 }
8989 let result = cross_block_alignr_256x1(
8990 self,
8991 Bytes::to_bytes(b).val.0,
8992 Bytes::to_bytes(a).val.0,
8993 SHIFT,
8994 );
8995 Bytes::from_bytes(u8x32 {
8996 val: crate::support::Aligned256(result),
8997 simd: self,
8998 })
8999 }
9000 #[inline(always)]
9001 fn slide_within_blocks_u8x32<const SHIFT: usize>(
9002 self,
9003 a: u8x32<Self>,
9004 b: u8x32<Self>,
9005 ) -> u8x32<Self> {
9006 if SHIFT >= 16usize {
9007 return b;
9008 }
9009 let result = dyn_alignr_256(
9010 self,
9011 Bytes::to_bytes(b).val.0,
9012 Bytes::to_bytes(a).val.0,
9013 SHIFT,
9014 );
9015 Bytes::from_bytes(u8x32 {
9016 val: crate::support::Aligned256(result),
9017 simd: self,
9018 })
9019 }
9020 #[inline(always)]
9021 fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
9022 crate::kernel!(
9023 #[inline(always)]
9024 fn kernel(token: Avx2, a: u8x32<Avx2>, indices: u8x32<Avx2>) -> u8x32<Avx2> {
9025 let result = _mm256_shuffle_epi8(Bytes::to_bytes(a).val.0, indices.into());
9026 Bytes::from_bytes(u8x32 {
9027 val: crate::support::Aligned256(result),
9028 simd: token,
9029 })
9030 }
9031 );
9032 kernel(self, a, indices)
9033 }
9034 #[inline(always)]
9035 fn swizzle_dyn_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
9036 crate::kernel!(
9037 #[inline(always)]
9038 fn kernel(token: Avx2, a: u8x32<Avx2>, indices: u8x32<Avx2>) -> u8x32<Avx2> {
9039 let bytes = Bytes::to_bytes(a).val.0;
9040 let indices = indices.into();
9041 let swapped = _mm256_permute2x128_si256::<0x01>(bytes, bytes);
9042 let lane_bias = _mm256_set_m128i(_mm_set1_epi8(-16), _mm_set1_epi8(112));
9043 let local_control = _mm256_add_epi8(indices, lane_bias);
9044 let remote_control = _mm256_xor_si256(local_control, _mm256_set1_epi8(i8::MIN));
9045 let local = _mm256_shuffle_epi8(bytes, local_control);
9046 let remote = _mm256_shuffle_epi8(swapped, remote_control);
9047 let result = _mm256_or_si256(local, remote);
9048 Bytes::from_bytes(u8x32 {
9049 val: crate::support::Aligned256(result),
9050 simd: token,
9051 })
9052 }
9053 );
9054 kernel(self, a, indices)
9055 }
9056 #[inline(always)]
9057 fn swizzle_dyn_precise_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
9058 crate::kernel!(
9059 #[inline(always)]
9060 fn kernel(token: Avx2, a: u8x32<Avx2>, indices: u8x32<Avx2>) -> u8x32<Avx2> {
9061 let bytes = Bytes::to_bytes(a);
9062 let indices = indices.into();
9063 let swapped = _mm256_permute2x128_si256::<0x01>(bytes.val.0, bytes.val.0);
9064 let control = _mm256_adds_epu8(indices, _mm256_set1_epi8(0x60));
9065 let local = _mm256_shuffle_epi8(bytes.val.0, control);
9066 let remote = _mm256_shuffle_epi8(swapped, control);
9067 let select_bias = _mm256_set_m128i(_mm_set1_epi8(-112), _mm_set1_epi8(16));
9068 let select_remote = _mm256_add_epi8(control, select_bias);
9069 let result = _mm256_blendv_epi8(local, remote, select_remote);
9070 let result_bytes = u8x32 {
9071 val: crate::support::Aligned256(result),
9072 simd: token,
9073 };
9074 Bytes::from_bytes(result_bytes)
9075 }
9076 );
9077 kernel(self, a, indices)
9078 }
9079 #[inline(always)]
9080 fn concat_swizzle_dyn_u8x32(
9081 self,
9082 a: u8x32<Self>,
9083 b: u8x32<Self>,
9084 indices: u8x32<Self>,
9085 ) -> u8x32<Self> {
9086 self.concat_swizzle_dyn_precise_u8x32(a, b, indices)
9087 }
9088 #[inline(always)]
9089 fn concat_swizzle_dyn_precise_u8x32(
9090 self,
9091 a: u8x32<Self>,
9092 b: u8x32<Self>,
9093 indices: u8x32<Self>,
9094 ) -> u8x32<Self> {
9095 crate::kernel!(
9096 #[inline(always)]
9097 fn kernel(
9098 token: Avx2,
9099 a: u8x32<Avx2>,
9100 b: u8x32<Avx2>,
9101 indices: u8x32<Avx2>,
9102 ) -> u8x32<Avx2> {
9103 let result = {
9104 let a = Bytes::to_bytes(a).val.0;
9105 let b = Bytes::to_bytes(b).val.0;
9106 let indices = indices.into();
9107 let control = _mm256_adds_epu8(indices, _mm256_set1_epi8(0x40));
9108 let a_swapped = _mm256_permute2x128_si256::<0x01>(a, a);
9109 let b_swapped = _mm256_permute2x128_si256::<0x01>(b, b);
9110 let flip_high_lane =
9111 _mm256_set_m128i(_mm_set1_epi8(i8::MIN), _mm_setzero_si128());
9112 let select_remote =
9113 _mm256_xor_si256(_mm256_slli_epi16::<3>(control), flip_high_lane);
9114 let from_a = _mm256_blendv_epi8(
9115 _mm256_shuffle_epi8(a, control),
9116 _mm256_shuffle_epi8(a_swapped, control),
9117 select_remote,
9118 );
9119 let from_b = _mm256_blendv_epi8(
9120 _mm256_shuffle_epi8(b, control),
9121 _mm256_shuffle_epi8(b_swapped, control),
9122 select_remote,
9123 );
9124 _mm256_blendv_epi8(from_a, from_b, _mm256_slli_epi16::<2>(control))
9125 };
9126 let result_bytes = u8x32 {
9127 val: crate::support::Aligned256(result),
9128 simd: token,
9129 };
9130 Bytes::from_bytes(result_bytes)
9131 }
9132 );
9133 kernel(self, a, b, indices)
9134 }
9135 #[inline(always)]
9136 fn compress_u8x32(self, values: u8x32<Self>, mask: mask8x32<Self>) -> u8x32<Self> {
9137 crate::kernel!(
9138 #[inline(always)]
9139 fn kernel(token: Avx2, values: u8x32<Avx2>, mask: mask8x32<Avx2>) -> u8x32<Avx2> {
9140 let mask_bits = token.to_bitmask_mask8x32(mask);
9141 let block_bits_0 = (mask_bits & 0xffff) as usize;
9142 let low_mask_0 = (block_bits_0) & 0xff;
9143 let high_mask_0 = (block_bits_0) >> 8;
9144 let low_control_0 = crate::support::COMPRESS_8_CONTROLS[low_mask_0];
9145 let high_control_0 = crate::support::COMPRESS_8_CONTROLS[high_mask_0];
9146 let high_control_0 = high_control_0 | 0x0808_0808_0808_0808;
9147 let control_0 =
9148 _mm_set_epi64x(high_control_0.cast_signed(), low_control_0.cast_signed());
9149 let low_count_0 = low_mask_0.count_ones() as usize;
9150 let block_bits_1 = (mask_bits >> 16 & 0xffff) as usize;
9151 let low_mask_1 = (block_bits_1) & 0xff;
9152 let high_mask_1 = (block_bits_1) >> 8;
9153 let low_control_1 = crate::support::COMPRESS_8_CONTROLS[low_mask_1];
9154 let high_control_1 = crate::support::COMPRESS_8_CONTROLS[high_mask_1];
9155 let high_control_1 = high_control_1 | 0x0808_0808_0808_0808;
9156 let control_1 =
9157 _mm_set_epi64x(high_control_1.cast_signed(), low_control_1.cast_signed());
9158 let low_count_1 = low_mask_1.count_ones() as usize;
9159 let control = _mm256_set_m128i(control_1, control_0);
9160 let compacted = _mm256_shuffle_epi8(values.val.0, control);
9161 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_0];
9162 let splice_0 = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
9163 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_1];
9164 let splice_1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
9165 let splice = _mm256_set_m128i(splice_1, splice_0);
9166 let compressed = _mm256_shuffle_epi8(compacted, splice);
9167 let splice = crate::transmute::checked_transmute_copy(
9168 &crate::support::COMPACT_16_SPLICE_CONTROLS[block_bits_0.count_ones() as usize],
9169 );
9170 let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed);
9171 let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed);
9172 let compressed_0 =
9173 _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice));
9174 let result_0 = compressed_0;
9175 result_0.simd_into(token)
9176 }
9177 );
9178 kernel(self, values, mask)
9179 }
9180 #[inline(always)]
9181 fn compress_merge_u8x32(
9182 self,
9183 values: u8x32<Self>,
9184 mask: mask8x32<Self>,
9185 merge: u8x32<Self>,
9186 ) -> u8x32<Self> {
9187 crate::kernel!(
9188 #[inline(always)]
9189 fn kernel(
9190 token: Avx2,
9191 values: u8x32<Avx2>,
9192 mask: mask8x32<Avx2>,
9193 merge: u8x32<Avx2>,
9194 ) -> u8x32<Avx2> {
9195 let mask_bits = token.to_bitmask_mask8x32(mask);
9196 let block_bits_0 = (mask_bits & 0xffff) as usize;
9197 let low_mask_0 = (block_bits_0) & 0xff;
9198 let high_mask_0 = (block_bits_0) >> 8;
9199 let low_control_0 = crate::support::COMPRESS_8_CONTROLS[low_mask_0];
9200 let high_control_0 = crate::support::COMPRESS_8_CONTROLS[high_mask_0];
9201 let high_control_0 = high_control_0 | 0x0808_0808_0808_0808;
9202 let control_0 =
9203 _mm_set_epi64x(high_control_0.cast_signed(), low_control_0.cast_signed());
9204 let low_count_0 = low_mask_0.count_ones() as usize;
9205 let block_bits_1 = (mask_bits >> 16 & 0xffff) as usize;
9206 let low_mask_1 = (block_bits_1) & 0xff;
9207 let high_mask_1 = (block_bits_1) >> 8;
9208 let low_control_1 = crate::support::COMPRESS_8_CONTROLS[low_mask_1];
9209 let high_control_1 = crate::support::COMPRESS_8_CONTROLS[high_mask_1];
9210 let high_control_1 = high_control_1 | 0x0808_0808_0808_0808;
9211 let control_1 =
9212 _mm_set_epi64x(high_control_1.cast_signed(), low_control_1.cast_signed());
9213 let low_count_1 = low_mask_1.count_ones() as usize;
9214 let control = _mm256_set_m128i(control_1, control_0);
9215 let compacted = _mm256_shuffle_epi8(values.val.0, control);
9216 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_0];
9217 let splice_0 = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
9218 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_1];
9219 let splice_1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
9220 let splice = _mm256_set_m128i(splice_1, splice_0);
9221 let compressed = _mm256_shuffle_epi8(compacted, splice);
9222 let splice = crate::transmute::checked_transmute_copy(
9223 &crate::support::COMPACT_16_SPLICE_CONTROLS[block_bits_0.count_ones() as usize],
9224 );
9225 let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed);
9226 let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed);
9227 let compressed_0 =
9228 _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice));
9229 let output_lane = (block_bits_0.count_ones() + block_bits_1.count_ones()) as usize;
9230 let prefix = crate::transmute::checked_transmute_copy(
9231 &crate::support::COMPACT_PREFIX_MASKS[output_lane.min(32)],
9232 );
9233 let result_0 = _mm256_blendv_epi8(merge.val.0, compressed_0, prefix);
9234 result_0.simd_into(token)
9235 }
9236 );
9237 kernel(self, values, mask, merge)
9238 }
9239 #[inline(always)]
9240 fn expand_u8x32(self, values: u8x32<Self>, mask: mask8x32<Self>) -> u8x32<Self> {
9241 crate::kernel!(
9242 #[inline(always)]
9243 fn kernel(token: Avx2, values: u8x32<Avx2>, mask: mask8x32<Avx2>) -> u8x32<Avx2> {
9244 let input = <[u8; 32]>::from(values);
9245 let mut output = [0u8; 32];
9246 let mut input_lane = 0;
9247 let block_bits = _mm_movemask_epi8(_mm256_castsi256_si128(mask.val.0)) as usize;
9248 let low_mask = (block_bits) & 0xff;
9249 let high_mask = (block_bits) >> 8;
9250 let low_count = low_mask.count_ones() as u64;
9251 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
9252 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
9253 let high_base = low_count * 0x0101_0101_0101_0101;
9254 let high_control = high_control + high_base;
9255 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
9256 let packed = crate::transmute::checked_transmute_copy(
9257 input[input_lane..].first_chunk::<16>().unwrap(),
9258 );
9259 let expanded = _mm_shuffle_epi8(packed, control);
9260 let result = expanded;
9261 crate::transmute::checked_transmute_store(
9262 result,
9263 output[0..].first_chunk_mut::<16>().unwrap(),
9264 );
9265 input_lane += block_bits.count_ones() as usize;
9266 let block_bits =
9267 _mm_movemask_epi8(_mm256_extracti128_si256::<1>(mask.val.0)) as usize;
9268 let low_mask = (block_bits) & 0xff;
9269 let high_mask = (block_bits) >> 8;
9270 let low_count = low_mask.count_ones() as u64;
9271 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
9272 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
9273 let high_base = low_count * 0x0101_0101_0101_0101;
9274 let high_control = high_control + high_base;
9275 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
9276 let packed = crate::transmute::checked_transmute_copy(
9277 input[input_lane..].first_chunk::<16>().unwrap(),
9278 );
9279 let expanded = _mm_shuffle_epi8(packed, control);
9280 let result = expanded;
9281 crate::transmute::checked_transmute_store(
9282 result,
9283 output[16..].first_chunk_mut::<16>().unwrap(),
9284 );
9285 u8x32::simd_from(token, output)
9286 }
9287 );
9288 kernel(self, values, mask)
9289 }
9290 #[inline(always)]
9291 fn expand_merge_u8x32(
9292 self,
9293 values: u8x32<Self>,
9294 mask: mask8x32<Self>,
9295 merge: u8x32<Self>,
9296 ) -> u8x32<Self> {
9297 crate::kernel!(
9298 #[inline(always)]
9299 fn kernel(
9300 token: Avx2,
9301 values: u8x32<Avx2>,
9302 mask: mask8x32<Avx2>,
9303 merge: u8x32<Avx2>,
9304 ) -> u8x32<Avx2> {
9305 let input = <[u8; 32]>::from(values);
9306 let mut output = [0u8; 32];
9307 let mut input_lane = 0;
9308 let block_bits = _mm_movemask_epi8(_mm256_castsi256_si128(mask.val.0)) as usize;
9309 let low_mask = (block_bits) & 0xff;
9310 let high_mask = (block_bits) >> 8;
9311 let low_count = low_mask.count_ones() as u64;
9312 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
9313 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
9314 let high_base = low_count * 0x0101_0101_0101_0101;
9315 let high_control = high_control + high_base;
9316 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
9317 let packed = crate::transmute::checked_transmute_copy(
9318 input[input_lane..].first_chunk::<16>().unwrap(),
9319 );
9320 let expanded = _mm_shuffle_epi8(packed, control);
9321 let result = _mm_blendv_epi8(
9322 _mm256_castsi256_si128(merge.val.0),
9323 expanded,
9324 _mm256_castsi256_si128(mask.val.0),
9325 );
9326 crate::transmute::checked_transmute_store(
9327 result,
9328 output[0..].first_chunk_mut::<16>().unwrap(),
9329 );
9330 input_lane += block_bits.count_ones() as usize;
9331 let block_bits =
9332 _mm_movemask_epi8(_mm256_extracti128_si256::<1>(mask.val.0)) as usize;
9333 let low_mask = (block_bits) & 0xff;
9334 let high_mask = (block_bits) >> 8;
9335 let low_count = low_mask.count_ones() as u64;
9336 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
9337 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
9338 let high_base = low_count * 0x0101_0101_0101_0101;
9339 let high_control = high_control + high_base;
9340 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
9341 let packed = crate::transmute::checked_transmute_copy(
9342 input[input_lane..].first_chunk::<16>().unwrap(),
9343 );
9344 let expanded = _mm_shuffle_epi8(packed, control);
9345 let result = _mm_blendv_epi8(
9346 _mm256_extracti128_si256::<1>(merge.val.0),
9347 expanded,
9348 _mm256_extracti128_si256::<1>(mask.val.0),
9349 );
9350 crate::transmute::checked_transmute_store(
9351 result,
9352 output[16..].first_chunk_mut::<16>().unwrap(),
9353 );
9354 u8x32::simd_from(token, output)
9355 }
9356 );
9357 kernel(self, values, mask, merge)
9358 }
9359 #[inline(always)]
9360 fn count_ones_u8x32(self, a: u8x32<Self>) -> u8x32<Self> {
9361 crate::kernel!(
9362 #[inline(always)]
9363 fn kernel(token: Avx2, a: u8x32<Avx2>) -> u8x32<Avx2> {
9364 let value = a.into();
9365 let nibble_mask = _mm256_set1_epi8(0x0f);
9366 let lookup = _mm256_setr_epi8(
9367 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 0, 1, 1, 2, 1, 2, 2, 3, 1, 2,
9368 2, 3, 2, 3, 3, 4,
9369 );
9370 let low = _mm256_and_si256(value, nibble_mask);
9371 let high = _mm256_and_si256(_mm256_srli_epi16::<4>(value), nibble_mask);
9372 let byte_counts = _mm256_add_epi8(
9373 _mm256_shuffle_epi8(lookup, low),
9374 _mm256_shuffle_epi8(lookup, high),
9375 );
9376 byte_counts.simd_into(token)
9377 }
9378 );
9379 kernel(self, a)
9380 }
9381 #[inline(always)]
9382 fn count_zeros_u8x32(self, a: u8x32<Self>) -> u8x32<Self> {
9383 self.count_ones_u8x32(self.not_u8x32(a))
9384 }
9385 #[inline(always)]
9386 fn add_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9387 crate::kernel!(
9388 #[inline(always)]
9389 fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
9390 _mm256_add_epi8(a.into(), b.into()).simd_into(token)
9391 }
9392 );
9393 kernel(self, a, b)
9394 }
9395 #[inline(always)]
9396 fn saturating_add_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9397 crate::kernel!(
9398 #[inline(always)]
9399 fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
9400 _mm256_adds_epu8(a.into(), b.into()).simd_into(token)
9401 }
9402 );
9403 kernel(self, a, b)
9404 }
9405 #[inline(always)]
9406 fn sub_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9407 crate::kernel!(
9408 #[inline(always)]
9409 fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
9410 _mm256_sub_epi8(a.into(), b.into()).simd_into(token)
9411 }
9412 );
9413 kernel(self, a, b)
9414 }
9415 #[inline(always)]
9416 fn saturating_sub_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9417 crate::kernel!(
9418 #[inline(always)]
9419 fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
9420 _mm256_subs_epu8(a.into(), b.into()).simd_into(token)
9421 }
9422 );
9423 kernel(self, a, b)
9424 }
9425 #[inline(always)]
9426 fn mul_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9427 crate::kernel!(
9428 #[inline(always)]
9429 fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
9430 let a = a.into();
9431 let b = b.into();
9432 let low_mask = _mm256_set1_epi16(0xFF);
9433 let dst_even = _mm256_mullo_epi16(a, b);
9434 let dst_odd = _mm256_maddubs_epi16(a, _mm256_andnot_si256(low_mask, b));
9435 _mm256_or_si256(
9436 _mm256_slli_epi16(dst_odd, 8),
9437 _mm256_and_si256(dst_even, low_mask),
9438 )
9439 .simd_into(token)
9440 }
9441 );
9442 kernel(self, a, b)
9443 }
9444 #[inline(always)]
9445 fn and_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9446 crate::kernel!(
9447 #[inline(always)]
9448 fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
9449 _mm256_and_si256(a.into(), b.into()).simd_into(token)
9450 }
9451 );
9452 kernel(self, a, b)
9453 }
9454 #[inline(always)]
9455 fn or_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9456 crate::kernel!(
9457 #[inline(always)]
9458 fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
9459 _mm256_or_si256(a.into(), b.into()).simd_into(token)
9460 }
9461 );
9462 kernel(self, a, b)
9463 }
9464 #[inline(always)]
9465 fn xor_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9466 crate::kernel!(
9467 #[inline(always)]
9468 fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
9469 _mm256_xor_si256(a.into(), b.into()).simd_into(token)
9470 }
9471 );
9472 kernel(self, a, b)
9473 }
9474 #[inline(always)]
9475 fn not_u8x32(self, a: u8x32<Self>) -> u8x32<Self> {
9476 a ^ !0
9477 }
9478 #[inline(always)]
9479 fn shl_u8x32(self, a: u8x32<Self>, shift: u32) -> u8x32<Self> {
9480 crate::kernel!(
9481 #[inline(always)]
9482 fn kernel(token: Avx2, a: u8x32<Avx2>, shift: u32) -> u8x32<Avx2> {
9483 let val = a.into();
9484 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
9485 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
9486 let byte_mask = _mm256_set1_epi8(mask_byte);
9487 _mm256_sll_epi16(_mm256_and_si256(val, byte_mask), shift_count).simd_into(token)
9488 }
9489 );
9490 kernel(self, a, shift)
9491 }
9492 #[inline(always)]
9493 fn shlv_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9494 [
9495 u8::wrapping_shl(a[0usize], b[0usize] as u32),
9496 u8::wrapping_shl(a[1usize], b[1usize] as u32),
9497 u8::wrapping_shl(a[2usize], b[2usize] as u32),
9498 u8::wrapping_shl(a[3usize], b[3usize] as u32),
9499 u8::wrapping_shl(a[4usize], b[4usize] as u32),
9500 u8::wrapping_shl(a[5usize], b[5usize] as u32),
9501 u8::wrapping_shl(a[6usize], b[6usize] as u32),
9502 u8::wrapping_shl(a[7usize], b[7usize] as u32),
9503 u8::wrapping_shl(a[8usize], b[8usize] as u32),
9504 u8::wrapping_shl(a[9usize], b[9usize] as u32),
9505 u8::wrapping_shl(a[10usize], b[10usize] as u32),
9506 u8::wrapping_shl(a[11usize], b[11usize] as u32),
9507 u8::wrapping_shl(a[12usize], b[12usize] as u32),
9508 u8::wrapping_shl(a[13usize], b[13usize] as u32),
9509 u8::wrapping_shl(a[14usize], b[14usize] as u32),
9510 u8::wrapping_shl(a[15usize], b[15usize] as u32),
9511 u8::wrapping_shl(a[16usize], b[16usize] as u32),
9512 u8::wrapping_shl(a[17usize], b[17usize] as u32),
9513 u8::wrapping_shl(a[18usize], b[18usize] as u32),
9514 u8::wrapping_shl(a[19usize], b[19usize] as u32),
9515 u8::wrapping_shl(a[20usize], b[20usize] as u32),
9516 u8::wrapping_shl(a[21usize], b[21usize] as u32),
9517 u8::wrapping_shl(a[22usize], b[22usize] as u32),
9518 u8::wrapping_shl(a[23usize], b[23usize] as u32),
9519 u8::wrapping_shl(a[24usize], b[24usize] as u32),
9520 u8::wrapping_shl(a[25usize], b[25usize] as u32),
9521 u8::wrapping_shl(a[26usize], b[26usize] as u32),
9522 u8::wrapping_shl(a[27usize], b[27usize] as u32),
9523 u8::wrapping_shl(a[28usize], b[28usize] as u32),
9524 u8::wrapping_shl(a[29usize], b[29usize] as u32),
9525 u8::wrapping_shl(a[30usize], b[30usize] as u32),
9526 u8::wrapping_shl(a[31usize], b[31usize] as u32),
9527 ]
9528 .simd_into(self)
9529 }
9530 #[inline(always)]
9531 fn shr_u8x32(self, a: u8x32<Self>, shift: u32) -> u8x32<Self> {
9532 crate::kernel!(
9533 #[inline(always)]
9534 fn kernel(token: Avx2, a: u8x32<Avx2>, shift: u32) -> u8x32<Avx2> {
9535 let val = a.into();
9536 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
9537 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
9538 let byte_mask = _mm256_set1_epi8(mask_byte);
9539 let shifted = _mm256_srl_epi16(val, shift_count);
9540 let result = { _mm256_and_si256(shifted, byte_mask) };
9541 result.simd_into(token)
9542 }
9543 );
9544 kernel(self, a, shift)
9545 }
9546 #[inline(always)]
9547 fn shrv_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9548 [
9549 u8::wrapping_shr(a[0usize], b[0usize] as u32),
9550 u8::wrapping_shr(a[1usize], b[1usize] as u32),
9551 u8::wrapping_shr(a[2usize], b[2usize] as u32),
9552 u8::wrapping_shr(a[3usize], b[3usize] as u32),
9553 u8::wrapping_shr(a[4usize], b[4usize] as u32),
9554 u8::wrapping_shr(a[5usize], b[5usize] as u32),
9555 u8::wrapping_shr(a[6usize], b[6usize] as u32),
9556 u8::wrapping_shr(a[7usize], b[7usize] as u32),
9557 u8::wrapping_shr(a[8usize], b[8usize] as u32),
9558 u8::wrapping_shr(a[9usize], b[9usize] as u32),
9559 u8::wrapping_shr(a[10usize], b[10usize] as u32),
9560 u8::wrapping_shr(a[11usize], b[11usize] as u32),
9561 u8::wrapping_shr(a[12usize], b[12usize] as u32),
9562 u8::wrapping_shr(a[13usize], b[13usize] as u32),
9563 u8::wrapping_shr(a[14usize], b[14usize] as u32),
9564 u8::wrapping_shr(a[15usize], b[15usize] as u32),
9565 u8::wrapping_shr(a[16usize], b[16usize] as u32),
9566 u8::wrapping_shr(a[17usize], b[17usize] as u32),
9567 u8::wrapping_shr(a[18usize], b[18usize] as u32),
9568 u8::wrapping_shr(a[19usize], b[19usize] as u32),
9569 u8::wrapping_shr(a[20usize], b[20usize] as u32),
9570 u8::wrapping_shr(a[21usize], b[21usize] as u32),
9571 u8::wrapping_shr(a[22usize], b[22usize] as u32),
9572 u8::wrapping_shr(a[23usize], b[23usize] as u32),
9573 u8::wrapping_shr(a[24usize], b[24usize] as u32),
9574 u8::wrapping_shr(a[25usize], b[25usize] as u32),
9575 u8::wrapping_shr(a[26usize], b[26usize] as u32),
9576 u8::wrapping_shr(a[27usize], b[27usize] as u32),
9577 u8::wrapping_shr(a[28usize], b[28usize] as u32),
9578 u8::wrapping_shr(a[29usize], b[29usize] as u32),
9579 u8::wrapping_shr(a[30usize], b[30usize] as u32),
9580 u8::wrapping_shr(a[31usize], b[31usize] as u32),
9581 ]
9582 .simd_into(self)
9583 }
9584 #[inline(always)]
9585 fn max_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9586 crate::kernel!(
9587 #[inline(always)]
9588 fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
9589 _mm256_max_epu8(a.into(), b.into()).simd_into(token)
9590 }
9591 );
9592 kernel(self, a, b)
9593 }
9594 #[inline(always)]
9595 fn min_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9596 crate::kernel!(
9597 #[inline(always)]
9598 fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
9599 _mm256_min_epu8(a.into(), b.into()).simd_into(token)
9600 }
9601 );
9602 kernel(self, a, b)
9603 }
9604 #[inline(always)]
9605 fn clamp_u8x32(self, a: u8x32<Self>, b: u8x32<Self>, c: u8x32<Self>) -> u8x32<Self> {
9606 debug_assert!(
9607 b.simd_le(c).all_true(),
9608 "clamp requires min <= max for all elements"
9609 );
9610 a.max(b).min(c)
9611 }
9612 #[inline(always)]
9613 fn simd_eq_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> mask8x32<Self> {
9614 crate::kernel!(
9615 #[inline(always)]
9616 fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> mask8x32<Avx2> {
9617 _mm256_cmpeq_epi8(a.into(), b.into()).simd_into(token)
9618 }
9619 );
9620 kernel(self, a, b)
9621 }
9622 #[inline(always)]
9623 fn simd_lt_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> mask8x32<Self> {
9624 crate::kernel!(
9625 #[inline(always)]
9626 fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> mask8x32<Avx2> {
9627 {
9628 let sign_bit = _mm256_set1_epi8(0x80u8.cast_signed());
9629 let lhs_signed = _mm256_xor_si256(b.into(), sign_bit);
9630 let rhs_signed = _mm256_xor_si256(a.into(), sign_bit);
9631 _mm256_cmpgt_epi8(lhs_signed, rhs_signed)
9632 }
9633 .simd_into(token)
9634 }
9635 );
9636 kernel(self, a, b)
9637 }
9638 #[inline(always)]
9639 fn simd_le_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> mask8x32<Self> {
9640 crate::kernel!(
9641 #[inline(always)]
9642 fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> mask8x32<Avx2> {
9643 _mm256_cmpeq_epi8(_mm256_min_epu8(a.into(), b.into()), a.into()).simd_into(token)
9644 }
9645 );
9646 kernel(self, a, b)
9647 }
9648 #[inline(always)]
9649 fn zip_low_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9650 crate::kernel!(
9651 #[inline(always)]
9652 fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
9653 let lo = _mm256_unpacklo_epi8(a.into(), b.into());
9654 let hi = _mm256_unpackhi_epi8(a.into(), b.into());
9655 _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token)
9656 }
9657 );
9658 kernel(self, a, b)
9659 }
9660 #[inline(always)]
9661 fn zip_high_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9662 crate::kernel!(
9663 #[inline(always)]
9664 fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
9665 let lo = _mm256_unpacklo_epi8(a.into(), b.into());
9666 let hi = _mm256_unpackhi_epi8(a.into(), b.into());
9667 _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token)
9668 }
9669 );
9670 kernel(self, a, b)
9671 }
9672 #[inline(always)]
9673 fn unzip_low_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9674 crate::kernel!(
9675 #[inline(always)]
9676 fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
9677 let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
9678 a.into(),
9679 _mm256_setr_epi8(
9680 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10,
9681 12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
9682 ),
9683 ));
9684 let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
9685 b.into(),
9686 _mm256_setr_epi8(
9687 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10,
9688 12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
9689 ),
9690 ));
9691 _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token)
9692 }
9693 );
9694 kernel(self, a, b)
9695 }
9696 #[inline(always)]
9697 fn unzip_high_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
9698 crate::kernel!(
9699 #[inline(always)]
9700 fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
9701 let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
9702 a.into(),
9703 _mm256_setr_epi8(
9704 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10,
9705 12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
9706 ),
9707 ));
9708 let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
9709 b.into(),
9710 _mm256_setr_epi8(
9711 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10,
9712 12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
9713 ),
9714 ));
9715 _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token)
9716 }
9717 );
9718 kernel(self, a, b)
9719 }
9720 #[inline(always)]
9721 fn interleave_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> (u8x32<Self>, u8x32<Self>) {
9722 crate::kernel!(
9723 #[inline(always)]
9724 fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> (u8x32<Avx2>, u8x32<Avx2>) {
9725 let lo = _mm256_unpacklo_epi8(a.into(), b.into());
9726 let hi = _mm256_unpackhi_epi8(a.into(), b.into());
9727 (
9728 _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token),
9729 _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token),
9730 )
9731 }
9732 );
9733 kernel(self, a, b)
9734 }
9735 #[inline(always)]
9736 fn deinterleave_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> (u8x32<Self>, u8x32<Self>) {
9737 crate::kernel!(
9738 #[inline(always)]
9739 fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> (u8x32<Avx2>, u8x32<Avx2>) {
9740 let mask = _mm256_setr_epi8(
9741 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, 12,
9742 14, 1, 3, 5, 7, 9, 11, 13, 15,
9743 );
9744 let a = _mm256_shuffle_epi8(a.into(), mask);
9745 let b = _mm256_shuffle_epi8(b.into(), mask);
9746 let low = _mm256_permute2x128_si256::<0b0010_0000>(a, b);
9747 let high = _mm256_permute2x128_si256::<0b0011_0001>(a, b);
9748 (
9749 _mm256_unpacklo_epi64(low, high).simd_into(token),
9750 _mm256_unpackhi_epi64(low, high).simd_into(token),
9751 )
9752 }
9753 );
9754 kernel(self, a, b)
9755 }
9756 #[inline(always)]
9757 fn select_u8x32(self, a: mask8x32<Self>, b: u8x32<Self>, c: u8x32<Self>) -> u8x32<Self> {
9758 crate::kernel!(
9759 #[inline(always)]
9760 fn kernel(
9761 token: Avx2,
9762 a: mask8x32<Avx2>,
9763 b: u8x32<Avx2>,
9764 c: u8x32<Avx2>,
9765 ) -> u8x32<Avx2> {
9766 _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
9767 }
9768 );
9769 kernel(self, a, b, c)
9770 }
9771 #[inline(always)]
9772 fn combine_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x64<Self> {
9773 u8x64 {
9774 val: crate::support::Aligned512([a.val.0, b.val.0]),
9775 simd: self,
9776 }
9777 }
9778 #[inline(always)]
9779 fn split_u8x32(self, a: u8x32<Self>) -> (u8x16<Self>, u8x16<Self>) {
9780 crate::kernel!(
9781 #[inline(always)]
9782 fn kernel(token: Avx2, a: u8x32<Avx2>) -> (u8x16<Avx2>, u8x16<Avx2>) {
9783 (
9784 _mm256_extracti128_si256::<0>(a.into()).simd_into(token),
9785 _mm256_extracti128_si256::<1>(a.into()).simd_into(token),
9786 )
9787 }
9788 );
9789 kernel(self, a)
9790 }
9791 #[inline(always)]
9792 fn widen_u8x32(self, a: u8x32<Self>) -> (u16x16<Self>, u16x16<Self>) {
9793 crate::kernel!(
9794 #[inline(always)]
9795 fn kernel(token: Avx2, a: u8x32<Avx2>) -> (u16x16<Avx2>, u16x16<Avx2>) {
9796 let raw = a.into();
9797 (
9798 _mm256_cvtepu8_epi16(_mm256_castsi256_si128(raw)).simd_into(token),
9799 _mm256_cvtepu8_epi16(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
9800 )
9801 }
9802 );
9803 kernel(self, a)
9804 }
9805 #[inline(always)]
9806 fn widening_mul_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> (u16x16<Self>, u16x16<Self>) {
9807 let (a0, a1) = self.widen_u8x32(a);
9808 let (b0, b1) = self.widen_u8x32(b);
9809 (self.mul_u16x16(a0, b0), self.mul_u16x16(a1, b1))
9810 }
9811 #[inline(always)]
9812 fn widening_add_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> (u16x16<Self>, u16x16<Self>) {
9813 crate::kernel!(
9814 #[inline(always)]
9815 fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> (u16x16<Avx2>, u16x16<Avx2>) {
9816 let a = a.into();
9817 let b = b.into();
9818 let low = a;
9819 let high = b;
9820 let result_low = _mm256_unpacklo_epi8(low, high);
9821 let result_high = _mm256_unpackhi_epi8(low, high);
9822 let ones = _mm256_set1_epi8(1);
9823 let result_low = _mm256_maddubs_epi16(result_low, ones);
9824 let result_high = _mm256_maddubs_epi16(result_high, ones);
9825 let lo = _mm256_permute2x128_si256::<0x20>(result_low, result_high);
9826 let hi = _mm256_permute2x128_si256::<0x31>(result_low, result_high);
9827 let result_low = lo;
9828 let result_high = hi;
9829 (result_low.simd_into(token), result_high.simd_into(token))
9830 }
9831 );
9832 kernel(self, a, b)
9833 }
9834 #[inline(always)]
9835 fn pairwise_widening_add_u8x32(self, a: u8x32<Self>) -> u16x16<Self> {
9836 crate::kernel!(
9837 #[inline(always)]
9838 fn kernel(token: Avx2, a: u8x32<Avx2>) -> u16x16<Avx2> {
9839 ({ _mm256_maddubs_epi16(a.into(), _mm256_set1_epi8(1)) }).simd_into(token)
9840 }
9841 );
9842 kernel(self, a)
9843 }
9844 #[inline(always)]
9845 fn pairwise_dot_product_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u16x16<Self> {
9846 let (low, high) = self.widening_mul_u8x32(a, b);
9847 let (even, odd) = self.deinterleave_u16x16(low, high);
9848 self.add_u16x16(even, odd)
9849 }
9850 #[inline(always)]
9851 fn splat_mask8x32(self, val: bool) -> mask8x32<Self> {
9852 crate::kernel!(
9853 #[inline(always)]
9854 fn kernel(token: Avx2, val: bool) -> mask8x32<Avx2> {
9855 let val: i8 = if val { !0 } else { 0 };
9856 _mm256_set1_epi8(val).simd_into(token)
9857 }
9858 );
9859 kernel(self, val)
9860 }
9861 #[inline(always)]
9862 fn from_bitmask_mask8x32(self, bits: u64) -> mask8x32<Self> {
9863 crate::kernel!(
9864 #[inline(always)]
9865 fn kernel(token: Avx2, bits: u64) -> mask8x32<Avx2> {
9866 {
9867 let bit_bytes = _mm256_broadcastsi128_si256(_mm_cvtsi32_si128(bits as i32));
9868 let bit_bytes = _mm256_shuffle_epi8(
9869 bit_bytes,
9870 _mm256_setr_epi8(
9871 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2,
9872 3, 3, 3, 3, 3, 3, 3, 3,
9873 ),
9874 );
9875 let bit_mask = _mm256_setr_epi8(
9876 1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16,
9877 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128,
9878 );
9879 _mm256_cmpeq_epi8(_mm256_and_si256(bit_bytes, bit_mask), bit_mask)
9880 }
9881 .simd_into(token)
9882 }
9883 );
9884 kernel(self, bits)
9885 }
9886 #[inline(always)]
9887 fn to_bitmask_mask8x32(self, a: mask8x32<Self>) -> u64 {
9888 crate::kernel!(
9889 #[inline(always)]
9890 fn kernel(token: Avx2, a: mask8x32<Avx2>) -> u64 {
9891 _mm256_movemask_epi8(a.into()) as u32 as u64
9892 }
9893 );
9894 kernel(self, a)
9895 }
9896 #[inline(always)]
9897 fn from_vector_mask8x32(self, vector: i8x32<Self>) -> mask8x32<Self> {
9898 mask8x32 {
9899 val: vector.val,
9900 simd: self,
9901 }
9902 }
9903 #[inline(always)]
9904 fn to_vector_mask8x32(self, a: mask8x32<Self>) -> i8x32<Self> {
9905 i8x32 {
9906 val: a.val,
9907 simd: self,
9908 }
9909 }
9910 #[inline(always)]
9911 fn set_mask8x32(self, a: &mut mask8x32<Self>, index: usize, value: bool) -> () {
9912 assert!(
9913 index < 32usize,
9914 "mask lane index {index} is out of bounds for {} lanes",
9915 32usize
9916 );
9917 let mut lanes: [i8; 32usize] = (*a).into();
9918 lanes[index] = if value { !0 } else { 0 };
9919 *a = lanes.simd_into(self);
9920 }
9921 #[inline(always)]
9922 fn rotate_elements_left_mask8x32<const OFFSET: usize>(
9923 self,
9924 a: mask8x32<Self>,
9925 ) -> mask8x32<Self> {
9926 let int = i8x32 {
9927 val: crate::transmute::checked_transmute_copy(&a.val),
9928 simd: self,
9929 };
9930 let rotated = int.rotate_elements_left::<OFFSET>();
9931 mask8x32 {
9932 val: crate::transmute::checked_transmute_copy(&rotated.val),
9933 simd: self,
9934 }
9935 }
9936 #[inline(always)]
9937 fn rotate_elements_right_mask8x32<const OFFSET: usize>(
9938 self,
9939 a: mask8x32<Self>,
9940 ) -> mask8x32<Self> {
9941 let int = i8x32 {
9942 val: crate::transmute::checked_transmute_copy(&a.val),
9943 simd: self,
9944 };
9945 let rotated = int.rotate_elements_right::<OFFSET>();
9946 mask8x32 {
9947 val: crate::transmute::checked_transmute_copy(&rotated.val),
9948 simd: self,
9949 }
9950 }
9951 #[inline(always)]
9952 fn and_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
9953 crate::kernel!(
9954 #[inline(always)]
9955 fn kernel(token: Avx2, a: mask8x32<Avx2>, b: mask8x32<Avx2>) -> mask8x32<Avx2> {
9956 _mm256_and_si256(a.into(), b.into()).simd_into(token)
9957 }
9958 );
9959 kernel(self, a, b)
9960 }
9961 #[inline(always)]
9962 fn or_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
9963 crate::kernel!(
9964 #[inline(always)]
9965 fn kernel(token: Avx2, a: mask8x32<Avx2>, b: mask8x32<Avx2>) -> mask8x32<Avx2> {
9966 _mm256_or_si256(a.into(), b.into()).simd_into(token)
9967 }
9968 );
9969 kernel(self, a, b)
9970 }
9971 #[inline(always)]
9972 fn xor_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
9973 crate::kernel!(
9974 #[inline(always)]
9975 fn kernel(token: Avx2, a: mask8x32<Avx2>, b: mask8x32<Avx2>) -> mask8x32<Avx2> {
9976 _mm256_xor_si256(a.into(), b.into()).simd_into(token)
9977 }
9978 );
9979 kernel(self, a, b)
9980 }
9981 #[inline(always)]
9982 fn not_mask8x32(self, a: mask8x32<Self>) -> mask8x32<Self> {
9983 self.xor_mask8x32(a, self.splat_mask8x32(true))
9984 }
9985 #[inline(always)]
9986 fn reverse_mask8x32(self, a: mask8x32<Self>) -> mask8x32<Self> {
9987 let lanes = i8x32 {
9988 val: crate::transmute::checked_transmute_copy(&a.val),
9989 simd: self,
9990 };
9991 let reversed = self.reverse_i8x32(lanes);
9992 mask8x32 {
9993 val: crate::transmute::checked_transmute_copy(&reversed.val),
9994 simd: self,
9995 }
9996 }
9997 #[inline(always)]
9998 fn select_mask8x32(
9999 self,
10000 a: mask8x32<Self>,
10001 b: mask8x32<Self>,
10002 c: mask8x32<Self>,
10003 ) -> mask8x32<Self> {
10004 crate::kernel!(
10005 #[inline(always)]
10006 fn kernel(
10007 token: Avx2,
10008 a: mask8x32<Avx2>,
10009 b: mask8x32<Avx2>,
10010 c: mask8x32<Avx2>,
10011 ) -> mask8x32<Avx2> {
10012 _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
10013 }
10014 );
10015 kernel(self, a, b, c)
10016 }
10017 #[inline(always)]
10018 fn simd_eq_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
10019 crate::kernel!(
10020 #[inline(always)]
10021 fn kernel(token: Avx2, a: mask8x32<Avx2>, b: mask8x32<Avx2>) -> mask8x32<Avx2> {
10022 _mm256_cmpeq_epi8(a.into(), b.into()).simd_into(token)
10023 }
10024 );
10025 kernel(self, a, b)
10026 }
10027 #[inline(always)]
10028 fn any_true_mask8x32(self, a: mask8x32<Self>) -> bool {
10029 crate::kernel!(
10030 #[inline(always)]
10031 fn kernel(token: Avx2, a: mask8x32<Avx2>) -> bool {
10032 _mm256_movemask_epi8(a.into()) as u32 != 0
10033 }
10034 );
10035 kernel(self, a)
10036 }
10037 #[inline(always)]
10038 fn all_true_mask8x32(self, a: mask8x32<Self>) -> bool {
10039 crate::kernel!(
10040 #[inline(always)]
10041 fn kernel(token: Avx2, a: mask8x32<Avx2>) -> bool {
10042 _mm256_movemask_epi8(a.into()) as u32 == 0xffffffff
10043 }
10044 );
10045 kernel(self, a)
10046 }
10047 #[inline(always)]
10048 fn any_false_mask8x32(self, a: mask8x32<Self>) -> bool {
10049 crate::kernel!(
10050 #[inline(always)]
10051 fn kernel(token: Avx2, a: mask8x32<Avx2>) -> bool {
10052 _mm256_movemask_epi8(a.into()) as u32 != 0xffffffff
10053 }
10054 );
10055 kernel(self, a)
10056 }
10057 #[inline(always)]
10058 fn all_false_mask8x32(self, a: mask8x32<Self>) -> bool {
10059 crate::kernel!(
10060 #[inline(always)]
10061 fn kernel(token: Avx2, a: mask8x32<Avx2>) -> bool {
10062 _mm256_movemask_epi8(a.into()) as u32 == 0
10063 }
10064 );
10065 kernel(self, a)
10066 }
10067 #[inline(always)]
10068 fn combine_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x64<Self> {
10069 mask8x64 {
10070 val: crate::support::Aligned512([a.val.0, b.val.0]),
10071 simd: self,
10072 }
10073 }
10074 #[inline(always)]
10075 fn split_mask8x32(self, a: mask8x32<Self>) -> (mask8x16<Self>, mask8x16<Self>) {
10076 crate::kernel!(
10077 #[inline(always)]
10078 fn kernel(token: Avx2, a: mask8x32<Avx2>) -> (mask8x16<Avx2>, mask8x16<Avx2>) {
10079 (
10080 _mm256_extracti128_si256::<0>(a.into()).simd_into(token),
10081 _mm256_extracti128_si256::<1>(a.into()).simd_into(token),
10082 )
10083 }
10084 );
10085 kernel(self, a)
10086 }
10087 #[inline(always)]
10088 fn widen_mask8x32(self, a: mask8x32<Self>) -> (mask16x16<Self>, mask16x16<Self>) {
10089 crate::kernel!(
10090 #[inline(always)]
10091 fn kernel(token: Avx2, a: mask8x32<Avx2>) -> (mask16x16<Avx2>, mask16x16<Avx2>) {
10092 let raw = a.into();
10093 (
10094 _mm256_cvtepi8_epi16(_mm256_castsi256_si128(raw)).simd_into(token),
10095 _mm256_cvtepi8_epi16(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
10096 )
10097 }
10098 );
10099 kernel(self, a)
10100 }
10101 #[inline(always)]
10102 fn abs_i16x16(self, a: i16x16<Self>) -> i16x16<Self> {
10103 crate::kernel!(
10104 #[inline(always)]
10105 fn kernel(token: Avx2, a: i16x16<Avx2>) -> i16x16<Avx2> {
10106 _mm256_abs_epi16(a.into()).simd_into(token)
10107 }
10108 );
10109 kernel(self, a)
10110 }
10111 #[inline(always)]
10112 fn splat_i16x16(self, val: i16) -> i16x16<Self> {
10113 crate::kernel!(
10114 #[inline(always)]
10115 fn kernel(token: Avx2, val: i16) -> i16x16<Avx2> {
10116 _mm256_set1_epi16(val).simd_into(token)
10117 }
10118 );
10119 kernel(self, val)
10120 }
10121 #[inline(always)]
10122 fn reverse_i16x16(self, a: i16x16<Self>) -> i16x16<Self> {
10123 let indices: u8x32<Self> = [
10124 30, 31, 28, 29, 26, 27, 24, 25, 22, 23, 20, 21, 18, 19, 16, 17, 14, 15, 12, 13, 10, 11,
10125 8, 9, 6, 7, 4, 5, 2, 3, 0, 1,
10126 ]
10127 .simd_into(self);
10128 self.swizzle_dyn_i16x16(a, indices)
10129 }
10130 #[inline(always)]
10131 fn slide_i16x16<const SHIFT: usize>(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10132 if SHIFT >= 16usize {
10133 return b;
10134 }
10135 let result = cross_block_alignr_256x1(
10136 self,
10137 Bytes::to_bytes(b).val.0,
10138 Bytes::to_bytes(a).val.0,
10139 SHIFT * 2usize,
10140 );
10141 Bytes::from_bytes(u8x32 {
10142 val: crate::support::Aligned256(result),
10143 simd: self,
10144 })
10145 }
10146 #[inline(always)]
10147 fn slide_within_blocks_i16x16<const SHIFT: usize>(
10148 self,
10149 a: i16x16<Self>,
10150 b: i16x16<Self>,
10151 ) -> i16x16<Self> {
10152 if SHIFT >= 8usize {
10153 return b;
10154 }
10155 let result = dyn_alignr_256(
10156 self,
10157 Bytes::to_bytes(b).val.0,
10158 Bytes::to_bytes(a).val.0,
10159 SHIFT * 2usize,
10160 );
10161 Bytes::from_bytes(u8x32 {
10162 val: crate::support::Aligned256(result),
10163 simd: self,
10164 })
10165 }
10166 #[inline(always)]
10167 fn compress_i16x16(self, values: i16x16<Self>, mask: mask16x16<Self>) -> i16x16<Self> {
10168 let mask = mask8x32 {
10169 val: crate::transmute::checked_transmute_copy(&mask.val),
10170 simd: self,
10171 };
10172 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
10173 }
10174 #[inline(always)]
10175 fn compress_merge_i16x16(
10176 self,
10177 values: i16x16<Self>,
10178 mask: mask16x16<Self>,
10179 merge: i16x16<Self>,
10180 ) -> i16x16<Self> {
10181 let mask = mask8x32 {
10182 val: crate::transmute::checked_transmute_copy(&mask.val),
10183 simd: self,
10184 };
10185 Bytes::from_bytes(self.compress_merge_u8x32(
10186 Bytes::to_bytes(values),
10187 mask,
10188 Bytes::to_bytes(merge),
10189 ))
10190 }
10191 #[inline(always)]
10192 fn expand_i16x16(self, values: i16x16<Self>, mask: mask16x16<Self>) -> i16x16<Self> {
10193 let mask = mask8x32 {
10194 val: crate::transmute::checked_transmute_copy(&mask.val),
10195 simd: self,
10196 };
10197 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
10198 }
10199 #[inline(always)]
10200 fn expand_merge_i16x16(
10201 self,
10202 values: i16x16<Self>,
10203 mask: mask16x16<Self>,
10204 merge: i16x16<Self>,
10205 ) -> i16x16<Self> {
10206 let mask = mask8x32 {
10207 val: crate::transmute::checked_transmute_copy(&mask.val),
10208 simd: self,
10209 };
10210 Bytes::from_bytes(self.expand_merge_u8x32(
10211 Bytes::to_bytes(values),
10212 mask,
10213 Bytes::to_bytes(merge),
10214 ))
10215 }
10216 #[inline(always)]
10217 fn count_ones_i16x16(self, a: i16x16<Self>) -> i16x16<Self> {
10218 crate::kernel!(
10219 #[inline(always)]
10220 fn kernel(token: Avx2, a: i16x16<Avx2>) -> i16x16<Avx2> {
10221 let value = a.into();
10222 let nibble_mask = _mm256_set1_epi8(0x0f);
10223 let lookup = _mm256_setr_epi8(
10224 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 0, 1, 1, 2, 1, 2, 2, 3, 1, 2,
10225 2, 3, 2, 3, 3, 4,
10226 );
10227 let low = _mm256_and_si256(value, nibble_mask);
10228 let high = _mm256_and_si256(_mm256_srli_epi16::<4>(value), nibble_mask);
10229 let byte_counts = _mm256_add_epi8(
10230 _mm256_shuffle_epi8(lookup, low),
10231 _mm256_shuffle_epi8(lookup, high),
10232 );
10233 _mm256_maddubs_epi16(byte_counts, _mm256_set1_epi8(1)).simd_into(token)
10234 }
10235 );
10236 kernel(self, a)
10237 }
10238 #[inline(always)]
10239 fn count_zeros_i16x16(self, a: i16x16<Self>) -> i16x16<Self> {
10240 self.count_ones_i16x16(self.not_i16x16(a))
10241 }
10242 #[inline(always)]
10243 fn add_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10244 crate::kernel!(
10245 #[inline(always)]
10246 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
10247 _mm256_add_epi16(a.into(), b.into()).simd_into(token)
10248 }
10249 );
10250 kernel(self, a, b)
10251 }
10252 #[inline(always)]
10253 fn saturating_add_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10254 crate::kernel!(
10255 #[inline(always)]
10256 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
10257 _mm256_adds_epi16(a.into(), b.into()).simd_into(token)
10258 }
10259 );
10260 kernel(self, a, b)
10261 }
10262 #[inline(always)]
10263 fn sub_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10264 crate::kernel!(
10265 #[inline(always)]
10266 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
10267 _mm256_sub_epi16(a.into(), b.into()).simd_into(token)
10268 }
10269 );
10270 kernel(self, a, b)
10271 }
10272 #[inline(always)]
10273 fn saturating_sub_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10274 crate::kernel!(
10275 #[inline(always)]
10276 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
10277 _mm256_subs_epi16(a.into(), b.into()).simd_into(token)
10278 }
10279 );
10280 kernel(self, a, b)
10281 }
10282 #[inline(always)]
10283 fn mul_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10284 crate::kernel!(
10285 #[inline(always)]
10286 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
10287 _mm256_mullo_epi16(a.into(), b.into()).simd_into(token)
10288 }
10289 );
10290 kernel(self, a, b)
10291 }
10292 #[inline(always)]
10293 fn and_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10294 crate::kernel!(
10295 #[inline(always)]
10296 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
10297 _mm256_and_si256(a.into(), b.into()).simd_into(token)
10298 }
10299 );
10300 kernel(self, a, b)
10301 }
10302 #[inline(always)]
10303 fn or_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10304 crate::kernel!(
10305 #[inline(always)]
10306 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
10307 _mm256_or_si256(a.into(), b.into()).simd_into(token)
10308 }
10309 );
10310 kernel(self, a, b)
10311 }
10312 #[inline(always)]
10313 fn xor_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10314 crate::kernel!(
10315 #[inline(always)]
10316 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
10317 _mm256_xor_si256(a.into(), b.into()).simd_into(token)
10318 }
10319 );
10320 kernel(self, a, b)
10321 }
10322 #[inline(always)]
10323 fn not_i16x16(self, a: i16x16<Self>) -> i16x16<Self> {
10324 a ^ !0
10325 }
10326 #[inline(always)]
10327 fn shl_i16x16(self, a: i16x16<Self>, shift: u32) -> i16x16<Self> {
10328 crate::kernel!(
10329 #[inline(always)]
10330 fn kernel(token: Avx2, a: i16x16<Avx2>, shift: u32) -> i16x16<Avx2> {
10331 _mm256_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
10332 }
10333 );
10334 kernel(self, a, shift)
10335 }
10336 #[inline(always)]
10337 fn shlv_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10338 [
10339 i16::wrapping_shl(a[0usize], b[0usize] as u32),
10340 i16::wrapping_shl(a[1usize], b[1usize] as u32),
10341 i16::wrapping_shl(a[2usize], b[2usize] as u32),
10342 i16::wrapping_shl(a[3usize], b[3usize] as u32),
10343 i16::wrapping_shl(a[4usize], b[4usize] as u32),
10344 i16::wrapping_shl(a[5usize], b[5usize] as u32),
10345 i16::wrapping_shl(a[6usize], b[6usize] as u32),
10346 i16::wrapping_shl(a[7usize], b[7usize] as u32),
10347 i16::wrapping_shl(a[8usize], b[8usize] as u32),
10348 i16::wrapping_shl(a[9usize], b[9usize] as u32),
10349 i16::wrapping_shl(a[10usize], b[10usize] as u32),
10350 i16::wrapping_shl(a[11usize], b[11usize] as u32),
10351 i16::wrapping_shl(a[12usize], b[12usize] as u32),
10352 i16::wrapping_shl(a[13usize], b[13usize] as u32),
10353 i16::wrapping_shl(a[14usize], b[14usize] as u32),
10354 i16::wrapping_shl(a[15usize], b[15usize] as u32),
10355 ]
10356 .simd_into(self)
10357 }
10358 #[inline(always)]
10359 fn shr_i16x16(self, a: i16x16<Self>, shift: u32) -> i16x16<Self> {
10360 crate::kernel!(
10361 #[inline(always)]
10362 fn kernel(token: Avx2, a: i16x16<Avx2>, shift: u32) -> i16x16<Avx2> {
10363 _mm256_sra_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
10364 }
10365 );
10366 kernel(self, a, shift)
10367 }
10368 #[inline(always)]
10369 fn shrv_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10370 [
10371 i16::wrapping_shr(a[0usize], b[0usize] as u32),
10372 i16::wrapping_shr(a[1usize], b[1usize] as u32),
10373 i16::wrapping_shr(a[2usize], b[2usize] as u32),
10374 i16::wrapping_shr(a[3usize], b[3usize] as u32),
10375 i16::wrapping_shr(a[4usize], b[4usize] as u32),
10376 i16::wrapping_shr(a[5usize], b[5usize] as u32),
10377 i16::wrapping_shr(a[6usize], b[6usize] as u32),
10378 i16::wrapping_shr(a[7usize], b[7usize] as u32),
10379 i16::wrapping_shr(a[8usize], b[8usize] as u32),
10380 i16::wrapping_shr(a[9usize], b[9usize] as u32),
10381 i16::wrapping_shr(a[10usize], b[10usize] as u32),
10382 i16::wrapping_shr(a[11usize], b[11usize] as u32),
10383 i16::wrapping_shr(a[12usize], b[12usize] as u32),
10384 i16::wrapping_shr(a[13usize], b[13usize] as u32),
10385 i16::wrapping_shr(a[14usize], b[14usize] as u32),
10386 i16::wrapping_shr(a[15usize], b[15usize] as u32),
10387 ]
10388 .simd_into(self)
10389 }
10390 #[inline(always)]
10391 fn max_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10392 crate::kernel!(
10393 #[inline(always)]
10394 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
10395 _mm256_max_epi16(a.into(), b.into()).simd_into(token)
10396 }
10397 );
10398 kernel(self, a, b)
10399 }
10400 #[inline(always)]
10401 fn min_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10402 crate::kernel!(
10403 #[inline(always)]
10404 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
10405 _mm256_min_epi16(a.into(), b.into()).simd_into(token)
10406 }
10407 );
10408 kernel(self, a, b)
10409 }
10410 #[inline(always)]
10411 fn clamp_i16x16(self, a: i16x16<Self>, b: i16x16<Self>, c: i16x16<Self>) -> i16x16<Self> {
10412 debug_assert!(
10413 b.simd_le(c).all_true(),
10414 "clamp requires min <= max for all elements"
10415 );
10416 a.max(b).min(c)
10417 }
10418 #[inline(always)]
10419 fn simd_eq_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> mask16x16<Self> {
10420 crate::kernel!(
10421 #[inline(always)]
10422 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> mask16x16<Avx2> {
10423 _mm256_cmpeq_epi16(a.into(), b.into()).simd_into(token)
10424 }
10425 );
10426 kernel(self, a, b)
10427 }
10428 #[inline(always)]
10429 fn simd_lt_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> mask16x16<Self> {
10430 crate::kernel!(
10431 #[inline(always)]
10432 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> mask16x16<Avx2> {
10433 _mm256_cmpgt_epi16(b.into(), a.into()).simd_into(token)
10434 }
10435 );
10436 kernel(self, a, b)
10437 }
10438 #[inline(always)]
10439 fn simd_le_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> mask16x16<Self> {
10440 crate::kernel!(
10441 #[inline(always)]
10442 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> mask16x16<Avx2> {
10443 _mm256_cmpeq_epi16(_mm256_min_epi16(a.into(), b.into()), a.into()).simd_into(token)
10444 }
10445 );
10446 kernel(self, a, b)
10447 }
10448 #[inline(always)]
10449 fn zip_low_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10450 crate::kernel!(
10451 #[inline(always)]
10452 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
10453 let lo = _mm256_unpacklo_epi16(a.into(), b.into());
10454 let hi = _mm256_unpackhi_epi16(a.into(), b.into());
10455 _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token)
10456 }
10457 );
10458 kernel(self, a, b)
10459 }
10460 #[inline(always)]
10461 fn zip_high_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10462 crate::kernel!(
10463 #[inline(always)]
10464 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
10465 let lo = _mm256_unpacklo_epi16(a.into(), b.into());
10466 let hi = _mm256_unpackhi_epi16(a.into(), b.into());
10467 _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token)
10468 }
10469 );
10470 kernel(self, a, b)
10471 }
10472 #[inline(always)]
10473 fn unzip_low_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10474 crate::kernel!(
10475 #[inline(always)]
10476 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
10477 let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
10478 a.into(),
10479 _mm256_setr_epi8(
10480 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12,
10481 13, 2, 3, 6, 7, 10, 11, 14, 15,
10482 ),
10483 ));
10484 let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
10485 b.into(),
10486 _mm256_setr_epi8(
10487 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12,
10488 13, 2, 3, 6, 7, 10, 11, 14, 15,
10489 ),
10490 ));
10491 _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token)
10492 }
10493 );
10494 kernel(self, a, b)
10495 }
10496 #[inline(always)]
10497 fn unzip_high_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
10498 crate::kernel!(
10499 #[inline(always)]
10500 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
10501 let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
10502 a.into(),
10503 _mm256_setr_epi8(
10504 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12,
10505 13, 2, 3, 6, 7, 10, 11, 14, 15,
10506 ),
10507 ));
10508 let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
10509 b.into(),
10510 _mm256_setr_epi8(
10511 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12,
10512 13, 2, 3, 6, 7, 10, 11, 14, 15,
10513 ),
10514 ));
10515 _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token)
10516 }
10517 );
10518 kernel(self, a, b)
10519 }
10520 #[inline(always)]
10521 fn interleave_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> (i16x16<Self>, i16x16<Self>) {
10522 crate::kernel!(
10523 #[inline(always)]
10524 fn kernel(
10525 token: Avx2,
10526 a: i16x16<Avx2>,
10527 b: i16x16<Avx2>,
10528 ) -> (i16x16<Avx2>, i16x16<Avx2>) {
10529 let lo = _mm256_unpacklo_epi16(a.into(), b.into());
10530 let hi = _mm256_unpackhi_epi16(a.into(), b.into());
10531 (
10532 _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token),
10533 _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token),
10534 )
10535 }
10536 );
10537 kernel(self, a, b)
10538 }
10539 #[inline(always)]
10540 fn deinterleave_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> (i16x16<Self>, i16x16<Self>) {
10541 crate::kernel!(
10542 #[inline(always)]
10543 fn kernel(
10544 token: Avx2,
10545 a: i16x16<Avx2>,
10546 b: i16x16<Avx2>,
10547 ) -> (i16x16<Avx2>, i16x16<Avx2>) {
10548 let mask = _mm256_setr_epi8(
10549 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12, 13,
10550 2, 3, 6, 7, 10, 11, 14, 15,
10551 );
10552 let a = _mm256_shuffle_epi8(a.into(), mask);
10553 let b = _mm256_shuffle_epi8(b.into(), mask);
10554 let low = _mm256_permute2x128_si256::<0b0010_0000>(a, b);
10555 let high = _mm256_permute2x128_si256::<0b0011_0001>(a, b);
10556 (
10557 _mm256_unpacklo_epi64(low, high).simd_into(token),
10558 _mm256_unpackhi_epi64(low, high).simd_into(token),
10559 )
10560 }
10561 );
10562 kernel(self, a, b)
10563 }
10564 #[inline(always)]
10565 fn select_i16x16(self, a: mask16x16<Self>, b: i16x16<Self>, c: i16x16<Self>) -> i16x16<Self> {
10566 crate::kernel!(
10567 #[inline(always)]
10568 fn kernel(
10569 token: Avx2,
10570 a: mask16x16<Avx2>,
10571 b: i16x16<Avx2>,
10572 c: i16x16<Avx2>,
10573 ) -> i16x16<Avx2> {
10574 _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
10575 }
10576 );
10577 kernel(self, a, b, c)
10578 }
10579 #[inline(always)]
10580 fn combine_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x32<Self> {
10581 i16x32 {
10582 val: crate::support::Aligned512([a.val.0, b.val.0]),
10583 simd: self,
10584 }
10585 }
10586 #[inline(always)]
10587 fn split_i16x16(self, a: i16x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
10588 crate::kernel!(
10589 #[inline(always)]
10590 fn kernel(token: Avx2, a: i16x16<Avx2>) -> (i16x8<Avx2>, i16x8<Avx2>) {
10591 (
10592 _mm256_extracti128_si256::<0>(a.into()).simd_into(token),
10593 _mm256_extracti128_si256::<1>(a.into()).simd_into(token),
10594 )
10595 }
10596 );
10597 kernel(self, a)
10598 }
10599 #[inline(always)]
10600 fn neg_i16x16(self, a: i16x16<Self>) -> i16x16<Self> {
10601 crate::kernel!(
10602 #[inline(always)]
10603 fn kernel(token: Avx2, a: i16x16<Avx2>) -> i16x16<Avx2> {
10604 _mm256_sub_epi16(_mm256_setzero_si256(), a.into()).simd_into(token)
10605 }
10606 );
10607 kernel(self, a)
10608 }
10609 #[inline(always)]
10610 fn widen_i16x16(self, a: i16x16<Self>) -> (i32x8<Self>, i32x8<Self>) {
10611 crate::kernel!(
10612 #[inline(always)]
10613 fn kernel(token: Avx2, a: i16x16<Avx2>) -> (i32x8<Avx2>, i32x8<Avx2>) {
10614 let raw = a.into();
10615 (
10616 _mm256_cvtepi16_epi32(_mm256_castsi256_si128(raw)).simd_into(token),
10617 _mm256_cvtepi16_epi32(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
10618 )
10619 }
10620 );
10621 kernel(self, a)
10622 }
10623 #[inline(always)]
10624 fn widening_mul_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> (i32x8<Self>, i32x8<Self>) {
10625 crate::kernel!(
10626 #[inline(always)]
10627 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> (i32x8<Avx2>, i32x8<Avx2>) {
10628 let a = a.into();
10629 let b = b.into();
10630 let low = _mm256_mullo_epi16(a, b);
10631 let high = _mm256_mulhi_epi16(a, b);
10632 let result_low = _mm256_unpacklo_epi16(low, high);
10633 let result_high = _mm256_unpackhi_epi16(low, high);
10634 let lo = _mm256_permute2x128_si256::<0x20>(result_low, result_high);
10635 let hi = _mm256_permute2x128_si256::<0x31>(result_low, result_high);
10636 let result_low = lo;
10637 let result_high = hi;
10638 (result_low.simd_into(token), result_high.simd_into(token))
10639 }
10640 );
10641 kernel(self, a, b)
10642 }
10643 #[inline(always)]
10644 fn widening_add_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> (i32x8<Self>, i32x8<Self>) {
10645 crate::kernel!(
10646 #[inline(always)]
10647 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> (i32x8<Avx2>, i32x8<Avx2>) {
10648 let a = a.into();
10649 let b = b.into();
10650 let low = a;
10651 let high = b;
10652 let result_low = _mm256_unpacklo_epi16(low, high);
10653 let result_high = _mm256_unpackhi_epi16(low, high);
10654 let ones = _mm256_set1_epi16(1);
10655 let result_low = _mm256_madd_epi16(result_low, ones);
10656 let result_high = _mm256_madd_epi16(result_high, ones);
10657 let lo = _mm256_permute2x128_si256::<0x20>(result_low, result_high);
10658 let hi = _mm256_permute2x128_si256::<0x31>(result_low, result_high);
10659 let result_low = lo;
10660 let result_high = hi;
10661 (result_low.simd_into(token), result_high.simd_into(token))
10662 }
10663 );
10664 kernel(self, a, b)
10665 }
10666 #[inline(always)]
10667 fn pairwise_widening_add_i16x16(self, a: i16x16<Self>) -> i32x8<Self> {
10668 crate::kernel!(
10669 #[inline(always)]
10670 fn kernel(token: Avx2, a: i16x16<Avx2>) -> i32x8<Avx2> {
10671 ({ _mm256_madd_epi16(a.into(), _mm256_set1_epi16(1)) }).simd_into(token)
10672 }
10673 );
10674 kernel(self, a)
10675 }
10676 #[inline(always)]
10677 fn pairwise_dot_product_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i32x8<Self> {
10678 crate::kernel!(
10679 #[inline(always)]
10680 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i32x8<Avx2> {
10681 _mm256_madd_epi16(a.into(), b.into()).simd_into(token)
10682 }
10683 );
10684 kernel(self, a, b)
10685 }
10686 #[inline(always)]
10687 fn narrow_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i8x32<Self> {
10688 crate::kernel!(
10689 #[inline(always)]
10690 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i8x32<Avx2> {
10691 _mm256_permute4x64_epi64::<0xd8>({
10692 let mask = _mm256_set1_epi16(0xff);
10693 _mm256_packus_epi16(
10694 _mm256_and_si256(a.into(), mask),
10695 _mm256_and_si256(b.into(), mask),
10696 )
10697 })
10698 .simd_into(token)
10699 }
10700 );
10701 kernel(self, a, b)
10702 }
10703 #[inline(always)]
10704 fn saturating_narrow_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i8x32<Self> {
10705 crate::kernel!(
10706 #[inline(always)]
10707 fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i8x32<Avx2> {
10708 _mm256_permute4x64_epi64::<0xd8>(_mm256_packs_epi16(a.into(), b.into()))
10709 .simd_into(token)
10710 }
10711 );
10712 kernel(self, a, b)
10713 }
10714 #[inline(always)]
10715 fn relaxed_narrow_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i8x32<Self> {
10716 debug_assert!(
10717 a.as_slice()
10718 .iter()
10719 .chain(b.as_slice())
10720 .all(|&value| { value >= i8::MIN as i16 && value <= i8::MAX as i16 }),
10721 "relaxed_narrow inputs must fit in the destination type",
10722 );
10723 self.saturating_narrow_i16x16(a, b)
10724 }
10725 #[inline(always)]
10726 fn splat_u16x16(self, val: u16) -> u16x16<Self> {
10727 crate::kernel!(
10728 #[inline(always)]
10729 fn kernel(token: Avx2, val: u16) -> u16x16<Avx2> {
10730 _mm256_set1_epi16(val.cast_signed()).simd_into(token)
10731 }
10732 );
10733 kernel(self, val)
10734 }
10735 #[inline(always)]
10736 fn reverse_u16x16(self, a: u16x16<Self>) -> u16x16<Self> {
10737 let indices: u8x32<Self> = [
10738 30, 31, 28, 29, 26, 27, 24, 25, 22, 23, 20, 21, 18, 19, 16, 17, 14, 15, 12, 13, 10, 11,
10739 8, 9, 6, 7, 4, 5, 2, 3, 0, 1,
10740 ]
10741 .simd_into(self);
10742 self.swizzle_dyn_u16x16(a, indices)
10743 }
10744 #[inline(always)]
10745 fn slide_u16x16<const SHIFT: usize>(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
10746 if SHIFT >= 16usize {
10747 return b;
10748 }
10749 let result = cross_block_alignr_256x1(
10750 self,
10751 Bytes::to_bytes(b).val.0,
10752 Bytes::to_bytes(a).val.0,
10753 SHIFT * 2usize,
10754 );
10755 Bytes::from_bytes(u8x32 {
10756 val: crate::support::Aligned256(result),
10757 simd: self,
10758 })
10759 }
10760 #[inline(always)]
10761 fn slide_within_blocks_u16x16<const SHIFT: usize>(
10762 self,
10763 a: u16x16<Self>,
10764 b: u16x16<Self>,
10765 ) -> u16x16<Self> {
10766 if SHIFT >= 8usize {
10767 return b;
10768 }
10769 let result = dyn_alignr_256(
10770 self,
10771 Bytes::to_bytes(b).val.0,
10772 Bytes::to_bytes(a).val.0,
10773 SHIFT * 2usize,
10774 );
10775 Bytes::from_bytes(u8x32 {
10776 val: crate::support::Aligned256(result),
10777 simd: self,
10778 })
10779 }
10780 #[inline(always)]
10781 fn compress_u16x16(self, values: u16x16<Self>, mask: mask16x16<Self>) -> u16x16<Self> {
10782 let mask = mask8x32 {
10783 val: crate::transmute::checked_transmute_copy(&mask.val),
10784 simd: self,
10785 };
10786 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
10787 }
10788 #[inline(always)]
10789 fn compress_merge_u16x16(
10790 self,
10791 values: u16x16<Self>,
10792 mask: mask16x16<Self>,
10793 merge: u16x16<Self>,
10794 ) -> u16x16<Self> {
10795 let mask = mask8x32 {
10796 val: crate::transmute::checked_transmute_copy(&mask.val),
10797 simd: self,
10798 };
10799 Bytes::from_bytes(self.compress_merge_u8x32(
10800 Bytes::to_bytes(values),
10801 mask,
10802 Bytes::to_bytes(merge),
10803 ))
10804 }
10805 #[inline(always)]
10806 fn expand_u16x16(self, values: u16x16<Self>, mask: mask16x16<Self>) -> u16x16<Self> {
10807 let mask = mask8x32 {
10808 val: crate::transmute::checked_transmute_copy(&mask.val),
10809 simd: self,
10810 };
10811 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
10812 }
10813 #[inline(always)]
10814 fn expand_merge_u16x16(
10815 self,
10816 values: u16x16<Self>,
10817 mask: mask16x16<Self>,
10818 merge: u16x16<Self>,
10819 ) -> u16x16<Self> {
10820 let mask = mask8x32 {
10821 val: crate::transmute::checked_transmute_copy(&mask.val),
10822 simd: self,
10823 };
10824 Bytes::from_bytes(self.expand_merge_u8x32(
10825 Bytes::to_bytes(values),
10826 mask,
10827 Bytes::to_bytes(merge),
10828 ))
10829 }
10830 #[inline(always)]
10831 fn count_ones_u16x16(self, a: u16x16<Self>) -> u16x16<Self> {
10832 crate::kernel!(
10833 #[inline(always)]
10834 fn kernel(token: Avx2, a: u16x16<Avx2>) -> u16x16<Avx2> {
10835 let value = a.into();
10836 let nibble_mask = _mm256_set1_epi8(0x0f);
10837 let lookup = _mm256_setr_epi8(
10838 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 0, 1, 1, 2, 1, 2, 2, 3, 1, 2,
10839 2, 3, 2, 3, 3, 4,
10840 );
10841 let low = _mm256_and_si256(value, nibble_mask);
10842 let high = _mm256_and_si256(_mm256_srli_epi16::<4>(value), nibble_mask);
10843 let byte_counts = _mm256_add_epi8(
10844 _mm256_shuffle_epi8(lookup, low),
10845 _mm256_shuffle_epi8(lookup, high),
10846 );
10847 _mm256_maddubs_epi16(byte_counts, _mm256_set1_epi8(1)).simd_into(token)
10848 }
10849 );
10850 kernel(self, a)
10851 }
10852 #[inline(always)]
10853 fn count_zeros_u16x16(self, a: u16x16<Self>) -> u16x16<Self> {
10854 self.count_ones_u16x16(self.not_u16x16(a))
10855 }
10856 #[inline(always)]
10857 fn add_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
10858 crate::kernel!(
10859 #[inline(always)]
10860 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
10861 _mm256_add_epi16(a.into(), b.into()).simd_into(token)
10862 }
10863 );
10864 kernel(self, a, b)
10865 }
10866 #[inline(always)]
10867 fn saturating_add_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
10868 crate::kernel!(
10869 #[inline(always)]
10870 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
10871 _mm256_adds_epu16(a.into(), b.into()).simd_into(token)
10872 }
10873 );
10874 kernel(self, a, b)
10875 }
10876 #[inline(always)]
10877 fn sub_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
10878 crate::kernel!(
10879 #[inline(always)]
10880 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
10881 _mm256_sub_epi16(a.into(), b.into()).simd_into(token)
10882 }
10883 );
10884 kernel(self, a, b)
10885 }
10886 #[inline(always)]
10887 fn saturating_sub_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
10888 crate::kernel!(
10889 #[inline(always)]
10890 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
10891 _mm256_subs_epu16(a.into(), b.into()).simd_into(token)
10892 }
10893 );
10894 kernel(self, a, b)
10895 }
10896 #[inline(always)]
10897 fn mul_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
10898 crate::kernel!(
10899 #[inline(always)]
10900 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
10901 _mm256_mullo_epi16(a.into(), b.into()).simd_into(token)
10902 }
10903 );
10904 kernel(self, a, b)
10905 }
10906 #[inline(always)]
10907 fn and_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
10908 crate::kernel!(
10909 #[inline(always)]
10910 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
10911 _mm256_and_si256(a.into(), b.into()).simd_into(token)
10912 }
10913 );
10914 kernel(self, a, b)
10915 }
10916 #[inline(always)]
10917 fn or_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
10918 crate::kernel!(
10919 #[inline(always)]
10920 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
10921 _mm256_or_si256(a.into(), b.into()).simd_into(token)
10922 }
10923 );
10924 kernel(self, a, b)
10925 }
10926 #[inline(always)]
10927 fn xor_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
10928 crate::kernel!(
10929 #[inline(always)]
10930 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
10931 _mm256_xor_si256(a.into(), b.into()).simd_into(token)
10932 }
10933 );
10934 kernel(self, a, b)
10935 }
10936 #[inline(always)]
10937 fn not_u16x16(self, a: u16x16<Self>) -> u16x16<Self> {
10938 a ^ !0
10939 }
10940 #[inline(always)]
10941 fn shl_u16x16(self, a: u16x16<Self>, shift: u32) -> u16x16<Self> {
10942 crate::kernel!(
10943 #[inline(always)]
10944 fn kernel(token: Avx2, a: u16x16<Avx2>, shift: u32) -> u16x16<Avx2> {
10945 _mm256_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
10946 }
10947 );
10948 kernel(self, a, shift)
10949 }
10950 #[inline(always)]
10951 fn shlv_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
10952 [
10953 u16::wrapping_shl(a[0usize], b[0usize] as u32),
10954 u16::wrapping_shl(a[1usize], b[1usize] as u32),
10955 u16::wrapping_shl(a[2usize], b[2usize] as u32),
10956 u16::wrapping_shl(a[3usize], b[3usize] as u32),
10957 u16::wrapping_shl(a[4usize], b[4usize] as u32),
10958 u16::wrapping_shl(a[5usize], b[5usize] as u32),
10959 u16::wrapping_shl(a[6usize], b[6usize] as u32),
10960 u16::wrapping_shl(a[7usize], b[7usize] as u32),
10961 u16::wrapping_shl(a[8usize], b[8usize] as u32),
10962 u16::wrapping_shl(a[9usize], b[9usize] as u32),
10963 u16::wrapping_shl(a[10usize], b[10usize] as u32),
10964 u16::wrapping_shl(a[11usize], b[11usize] as u32),
10965 u16::wrapping_shl(a[12usize], b[12usize] as u32),
10966 u16::wrapping_shl(a[13usize], b[13usize] as u32),
10967 u16::wrapping_shl(a[14usize], b[14usize] as u32),
10968 u16::wrapping_shl(a[15usize], b[15usize] as u32),
10969 ]
10970 .simd_into(self)
10971 }
10972 #[inline(always)]
10973 fn shr_u16x16(self, a: u16x16<Self>, shift: u32) -> u16x16<Self> {
10974 crate::kernel!(
10975 #[inline(always)]
10976 fn kernel(token: Avx2, a: u16x16<Avx2>, shift: u32) -> u16x16<Avx2> {
10977 _mm256_srl_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
10978 }
10979 );
10980 kernel(self, a, shift)
10981 }
10982 #[inline(always)]
10983 fn shrv_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
10984 [
10985 u16::wrapping_shr(a[0usize], b[0usize] as u32),
10986 u16::wrapping_shr(a[1usize], b[1usize] as u32),
10987 u16::wrapping_shr(a[2usize], b[2usize] as u32),
10988 u16::wrapping_shr(a[3usize], b[3usize] as u32),
10989 u16::wrapping_shr(a[4usize], b[4usize] as u32),
10990 u16::wrapping_shr(a[5usize], b[5usize] as u32),
10991 u16::wrapping_shr(a[6usize], b[6usize] as u32),
10992 u16::wrapping_shr(a[7usize], b[7usize] as u32),
10993 u16::wrapping_shr(a[8usize], b[8usize] as u32),
10994 u16::wrapping_shr(a[9usize], b[9usize] as u32),
10995 u16::wrapping_shr(a[10usize], b[10usize] as u32),
10996 u16::wrapping_shr(a[11usize], b[11usize] as u32),
10997 u16::wrapping_shr(a[12usize], b[12usize] as u32),
10998 u16::wrapping_shr(a[13usize], b[13usize] as u32),
10999 u16::wrapping_shr(a[14usize], b[14usize] as u32),
11000 u16::wrapping_shr(a[15usize], b[15usize] as u32),
11001 ]
11002 .simd_into(self)
11003 }
11004 #[inline(always)]
11005 fn max_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11006 crate::kernel!(
11007 #[inline(always)]
11008 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
11009 _mm256_max_epu16(a.into(), b.into()).simd_into(token)
11010 }
11011 );
11012 kernel(self, a, b)
11013 }
11014 #[inline(always)]
11015 fn min_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11016 crate::kernel!(
11017 #[inline(always)]
11018 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
11019 _mm256_min_epu16(a.into(), b.into()).simd_into(token)
11020 }
11021 );
11022 kernel(self, a, b)
11023 }
11024 #[inline(always)]
11025 fn clamp_u16x16(self, a: u16x16<Self>, b: u16x16<Self>, c: u16x16<Self>) -> u16x16<Self> {
11026 debug_assert!(
11027 b.simd_le(c).all_true(),
11028 "clamp requires min <= max for all elements"
11029 );
11030 a.max(b).min(c)
11031 }
11032 #[inline(always)]
11033 fn simd_eq_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> mask16x16<Self> {
11034 crate::kernel!(
11035 #[inline(always)]
11036 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> mask16x16<Avx2> {
11037 _mm256_cmpeq_epi16(a.into(), b.into()).simd_into(token)
11038 }
11039 );
11040 kernel(self, a, b)
11041 }
11042 #[inline(always)]
11043 fn simd_lt_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> mask16x16<Self> {
11044 crate::kernel!(
11045 #[inline(always)]
11046 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> mask16x16<Avx2> {
11047 {
11048 let sign_bit = _mm256_set1_epi16(0x8000u16.cast_signed());
11049 let lhs_signed = _mm256_xor_si256(b.into(), sign_bit);
11050 let rhs_signed = _mm256_xor_si256(a.into(), sign_bit);
11051 _mm256_cmpgt_epi16(lhs_signed, rhs_signed)
11052 }
11053 .simd_into(token)
11054 }
11055 );
11056 kernel(self, a, b)
11057 }
11058 #[inline(always)]
11059 fn simd_le_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> mask16x16<Self> {
11060 crate::kernel!(
11061 #[inline(always)]
11062 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> mask16x16<Avx2> {
11063 _mm256_cmpeq_epi16(_mm256_min_epu16(a.into(), b.into()), a.into()).simd_into(token)
11064 }
11065 );
11066 kernel(self, a, b)
11067 }
11068 #[inline(always)]
11069 fn zip_low_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11070 crate::kernel!(
11071 #[inline(always)]
11072 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
11073 let lo = _mm256_unpacklo_epi16(a.into(), b.into());
11074 let hi = _mm256_unpackhi_epi16(a.into(), b.into());
11075 _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token)
11076 }
11077 );
11078 kernel(self, a, b)
11079 }
11080 #[inline(always)]
11081 fn zip_high_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11082 crate::kernel!(
11083 #[inline(always)]
11084 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
11085 let lo = _mm256_unpacklo_epi16(a.into(), b.into());
11086 let hi = _mm256_unpackhi_epi16(a.into(), b.into());
11087 _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token)
11088 }
11089 );
11090 kernel(self, a, b)
11091 }
11092 #[inline(always)]
11093 fn unzip_low_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11094 crate::kernel!(
11095 #[inline(always)]
11096 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
11097 let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
11098 a.into(),
11099 _mm256_setr_epi8(
11100 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12,
11101 13, 2, 3, 6, 7, 10, 11, 14, 15,
11102 ),
11103 ));
11104 let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
11105 b.into(),
11106 _mm256_setr_epi8(
11107 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12,
11108 13, 2, 3, 6, 7, 10, 11, 14, 15,
11109 ),
11110 ));
11111 _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token)
11112 }
11113 );
11114 kernel(self, a, b)
11115 }
11116 #[inline(always)]
11117 fn unzip_high_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
11118 crate::kernel!(
11119 #[inline(always)]
11120 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
11121 let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
11122 a.into(),
11123 _mm256_setr_epi8(
11124 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12,
11125 13, 2, 3, 6, 7, 10, 11, 14, 15,
11126 ),
11127 ));
11128 let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
11129 b.into(),
11130 _mm256_setr_epi8(
11131 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12,
11132 13, 2, 3, 6, 7, 10, 11, 14, 15,
11133 ),
11134 ));
11135 _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token)
11136 }
11137 );
11138 kernel(self, a, b)
11139 }
11140 #[inline(always)]
11141 fn interleave_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> (u16x16<Self>, u16x16<Self>) {
11142 crate::kernel!(
11143 #[inline(always)]
11144 fn kernel(
11145 token: Avx2,
11146 a: u16x16<Avx2>,
11147 b: u16x16<Avx2>,
11148 ) -> (u16x16<Avx2>, u16x16<Avx2>) {
11149 let lo = _mm256_unpacklo_epi16(a.into(), b.into());
11150 let hi = _mm256_unpackhi_epi16(a.into(), b.into());
11151 (
11152 _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token),
11153 _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token),
11154 )
11155 }
11156 );
11157 kernel(self, a, b)
11158 }
11159 #[inline(always)]
11160 fn deinterleave_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> (u16x16<Self>, u16x16<Self>) {
11161 crate::kernel!(
11162 #[inline(always)]
11163 fn kernel(
11164 token: Avx2,
11165 a: u16x16<Avx2>,
11166 b: u16x16<Avx2>,
11167 ) -> (u16x16<Avx2>, u16x16<Avx2>) {
11168 let mask = _mm256_setr_epi8(
11169 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12, 13,
11170 2, 3, 6, 7, 10, 11, 14, 15,
11171 );
11172 let a = _mm256_shuffle_epi8(a.into(), mask);
11173 let b = _mm256_shuffle_epi8(b.into(), mask);
11174 let low = _mm256_permute2x128_si256::<0b0010_0000>(a, b);
11175 let high = _mm256_permute2x128_si256::<0b0011_0001>(a, b);
11176 (
11177 _mm256_unpacklo_epi64(low, high).simd_into(token),
11178 _mm256_unpackhi_epi64(low, high).simd_into(token),
11179 )
11180 }
11181 );
11182 kernel(self, a, b)
11183 }
11184 #[inline(always)]
11185 fn select_u16x16(self, a: mask16x16<Self>, b: u16x16<Self>, c: u16x16<Self>) -> u16x16<Self> {
11186 crate::kernel!(
11187 #[inline(always)]
11188 fn kernel(
11189 token: Avx2,
11190 a: mask16x16<Avx2>,
11191 b: u16x16<Avx2>,
11192 c: u16x16<Avx2>,
11193 ) -> u16x16<Avx2> {
11194 _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
11195 }
11196 );
11197 kernel(self, a, b, c)
11198 }
11199 #[inline(always)]
11200 fn combine_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x32<Self> {
11201 u16x32 {
11202 val: crate::support::Aligned512([a.val.0, b.val.0]),
11203 simd: self,
11204 }
11205 }
11206 #[inline(always)]
11207 fn split_u16x16(self, a: u16x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
11208 crate::kernel!(
11209 #[inline(always)]
11210 fn kernel(token: Avx2, a: u16x16<Avx2>) -> (u16x8<Avx2>, u16x8<Avx2>) {
11211 (
11212 _mm256_extracti128_si256::<0>(a.into()).simd_into(token),
11213 _mm256_extracti128_si256::<1>(a.into()).simd_into(token),
11214 )
11215 }
11216 );
11217 kernel(self, a)
11218 }
11219 #[inline(always)]
11220 fn widen_u16x16(self, a: u16x16<Self>) -> (u32x8<Self>, u32x8<Self>) {
11221 crate::kernel!(
11222 #[inline(always)]
11223 fn kernel(token: Avx2, a: u16x16<Avx2>) -> (u32x8<Avx2>, u32x8<Avx2>) {
11224 let raw = a.into();
11225 (
11226 _mm256_cvtepu16_epi32(_mm256_castsi256_si128(raw)).simd_into(token),
11227 _mm256_cvtepu16_epi32(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
11228 )
11229 }
11230 );
11231 kernel(self, a)
11232 }
11233 #[inline(always)]
11234 fn widening_mul_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> (u32x8<Self>, u32x8<Self>) {
11235 crate::kernel!(
11236 #[inline(always)]
11237 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> (u32x8<Avx2>, u32x8<Avx2>) {
11238 let a = a.into();
11239 let b = b.into();
11240 let low = _mm256_mullo_epi16(a, b);
11241 let high = _mm256_mulhi_epu16(a, b);
11242 let result_low = _mm256_unpacklo_epi16(low, high);
11243 let result_high = _mm256_unpackhi_epi16(low, high);
11244 let lo = _mm256_permute2x128_si256::<0x20>(result_low, result_high);
11245 let hi = _mm256_permute2x128_si256::<0x31>(result_low, result_high);
11246 let result_low = lo;
11247 let result_high = hi;
11248 (result_low.simd_into(token), result_high.simd_into(token))
11249 }
11250 );
11251 kernel(self, a, b)
11252 }
11253 #[inline(always)]
11254 fn widening_add_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> (u32x8<Self>, u32x8<Self>) {
11255 let (a0, a1) = self.widen_u16x16(a);
11256 let (b0, b1) = self.widen_u16x16(b);
11257 (self.add_u32x8(a0, b0), self.add_u32x8(a1, b1))
11258 }
11259 #[inline(always)]
11260 fn pairwise_widening_add_u16x16(self, a: u16x16<Self>) -> u32x8<Self> {
11261 crate::kernel!(
11262 #[inline(always)]
11263 fn kernel(token: Avx2, a: u16x16<Avx2>) -> u32x8<Avx2> {
11264 ({
11265 let a = a.into();
11266 _mm256_add_epi32(
11267 _mm256_and_si256(a, _mm256_set1_epi32(65535)),
11268 _mm256_srli_epi32::<16>(a),
11269 )
11270 })
11271 .simd_into(token)
11272 }
11273 );
11274 kernel(self, a)
11275 }
11276 #[inline(always)]
11277 fn pairwise_dot_product_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u32x8<Self> {
11278 let (low, high) = self.widening_mul_u16x16(a, b);
11279 let (even, odd) = self.deinterleave_u32x8(low, high);
11280 self.add_u32x8(even, odd)
11281 }
11282 #[inline(always)]
11283 fn narrow_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u8x32<Self> {
11284 crate::kernel!(
11285 #[inline(always)]
11286 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u8x32<Avx2> {
11287 _mm256_permute4x64_epi64::<0xd8>({
11288 let mask = _mm256_set1_epi16(0xff);
11289 _mm256_packus_epi16(
11290 _mm256_and_si256(a.into(), mask),
11291 _mm256_and_si256(b.into(), mask),
11292 )
11293 })
11294 .simd_into(token)
11295 }
11296 );
11297 kernel(self, a, b)
11298 }
11299 #[inline(always)]
11300 fn saturating_narrow_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u8x32<Self> {
11301 crate::kernel!(
11302 #[inline(always)]
11303 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u8x32<Avx2> {
11304 _mm256_permute4x64_epi64::<0xd8>({
11305 let max = _mm256_set1_epi16(i16::MAX);
11306 _mm256_packus_epi16(
11307 _mm256_min_epu16(a.into(), max),
11308 _mm256_min_epu16(b.into(), max),
11309 )
11310 })
11311 .simd_into(token)
11312 }
11313 );
11314 kernel(self, a, b)
11315 }
11316 #[inline(always)]
11317 fn relaxed_narrow_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u8x32<Self> {
11318 crate::kernel!(
11319 #[inline(always)]
11320 fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u8x32<Avx2> {
11321 debug_assert!(
11322 a.as_slice()
11323 .iter()
11324 .chain(b.as_slice())
11325 .all(|&value| { value >= u8::MIN as u16 && value <= u8::MAX as u16 }),
11326 "relaxed_narrow inputs must fit in the destination type",
11327 );
11328 _mm256_permute4x64_epi64::<0xd8>(_mm256_packus_epi16(a.into(), b.into()))
11329 .simd_into(token)
11330 }
11331 );
11332 kernel(self, a, b)
11333 }
11334 #[inline(always)]
11335 fn splat_mask16x16(self, val: bool) -> mask16x16<Self> {
11336 crate::kernel!(
11337 #[inline(always)]
11338 fn kernel(token: Avx2, val: bool) -> mask16x16<Avx2> {
11339 let val: i16 = if val { !0 } else { 0 };
11340 _mm256_set1_epi16(val).simd_into(token)
11341 }
11342 );
11343 kernel(self, val)
11344 }
11345 #[inline(always)]
11346 fn from_bitmask_mask16x16(self, bits: u64) -> mask16x16<Self> {
11347 crate::kernel!(
11348 #[inline(always)]
11349 fn kernel(token: Avx2, bits: u64) -> mask16x16<Avx2> {
11350 {
11351 let bit_lanes = _mm256_set1_epi16(bits as i16);
11352 let bit_mask = _mm256_setr_epi16(
11353 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384,
11354 -32768,
11355 );
11356 _mm256_cmpeq_epi16(_mm256_and_si256(bit_lanes, bit_mask), bit_mask)
11357 }
11358 .simd_into(token)
11359 }
11360 );
11361 kernel(self, bits)
11362 }
11363 #[inline(always)]
11364 fn to_bitmask_mask16x16(self, a: mask16x16<Self>) -> u64 {
11365 crate::kernel!(
11366 #[inline(always)]
11367 fn kernel(token: Avx2, a: mask16x16<Avx2>) -> u64 {
11368 {
11369 let halves: [__m128i; 2usize] =
11370 crate::transmute::checked_transmute_copy(&a.val.0);
11371 let packed = _mm_packs_epi16(halves[0], halves[1]);
11372 _mm_movemask_epi8(packed) as u32 as u64
11373 }
11374 }
11375 );
11376 kernel(self, a)
11377 }
11378 #[inline(always)]
11379 fn from_vector_mask16x16(self, vector: i16x16<Self>) -> mask16x16<Self> {
11380 mask16x16 {
11381 val: vector.val,
11382 simd: self,
11383 }
11384 }
11385 #[inline(always)]
11386 fn to_vector_mask16x16(self, a: mask16x16<Self>) -> i16x16<Self> {
11387 i16x16 {
11388 val: a.val,
11389 simd: self,
11390 }
11391 }
11392 #[inline(always)]
11393 fn set_mask16x16(self, a: &mut mask16x16<Self>, index: usize, value: bool) -> () {
11394 assert!(
11395 index < 16usize,
11396 "mask lane index {index} is out of bounds for {} lanes",
11397 16usize
11398 );
11399 let mut lanes: [i16; 16usize] = (*a).into();
11400 lanes[index] = if value { !0 } else { 0 };
11401 *a = lanes.simd_into(self);
11402 }
11403 #[inline(always)]
11404 fn rotate_elements_left_mask16x16<const OFFSET: usize>(
11405 self,
11406 a: mask16x16<Self>,
11407 ) -> mask16x16<Self> {
11408 let int = i16x16 {
11409 val: crate::transmute::checked_transmute_copy(&a.val),
11410 simd: self,
11411 };
11412 let rotated = int.rotate_elements_left::<OFFSET>();
11413 mask16x16 {
11414 val: crate::transmute::checked_transmute_copy(&rotated.val),
11415 simd: self,
11416 }
11417 }
11418 #[inline(always)]
11419 fn rotate_elements_right_mask16x16<const OFFSET: usize>(
11420 self,
11421 a: mask16x16<Self>,
11422 ) -> mask16x16<Self> {
11423 let int = i16x16 {
11424 val: crate::transmute::checked_transmute_copy(&a.val),
11425 simd: self,
11426 };
11427 let rotated = int.rotate_elements_right::<OFFSET>();
11428 mask16x16 {
11429 val: crate::transmute::checked_transmute_copy(&rotated.val),
11430 simd: self,
11431 }
11432 }
11433 #[inline(always)]
11434 fn and_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
11435 crate::kernel!(
11436 #[inline(always)]
11437 fn kernel(token: Avx2, a: mask16x16<Avx2>, b: mask16x16<Avx2>) -> mask16x16<Avx2> {
11438 _mm256_and_si256(a.into(), b.into()).simd_into(token)
11439 }
11440 );
11441 kernel(self, a, b)
11442 }
11443 #[inline(always)]
11444 fn or_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
11445 crate::kernel!(
11446 #[inline(always)]
11447 fn kernel(token: Avx2, a: mask16x16<Avx2>, b: mask16x16<Avx2>) -> mask16x16<Avx2> {
11448 _mm256_or_si256(a.into(), b.into()).simd_into(token)
11449 }
11450 );
11451 kernel(self, a, b)
11452 }
11453 #[inline(always)]
11454 fn xor_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
11455 crate::kernel!(
11456 #[inline(always)]
11457 fn kernel(token: Avx2, a: mask16x16<Avx2>, b: mask16x16<Avx2>) -> mask16x16<Avx2> {
11458 _mm256_xor_si256(a.into(), b.into()).simd_into(token)
11459 }
11460 );
11461 kernel(self, a, b)
11462 }
11463 #[inline(always)]
11464 fn not_mask16x16(self, a: mask16x16<Self>) -> mask16x16<Self> {
11465 self.xor_mask16x16(a, self.splat_mask16x16(true))
11466 }
11467 #[inline(always)]
11468 fn reverse_mask16x16(self, a: mask16x16<Self>) -> mask16x16<Self> {
11469 let lanes = i16x16 {
11470 val: crate::transmute::checked_transmute_copy(&a.val),
11471 simd: self,
11472 };
11473 let reversed = self.reverse_i16x16(lanes);
11474 mask16x16 {
11475 val: crate::transmute::checked_transmute_copy(&reversed.val),
11476 simd: self,
11477 }
11478 }
11479 #[inline(always)]
11480 fn select_mask16x16(
11481 self,
11482 a: mask16x16<Self>,
11483 b: mask16x16<Self>,
11484 c: mask16x16<Self>,
11485 ) -> mask16x16<Self> {
11486 crate::kernel!(
11487 #[inline(always)]
11488 fn kernel(
11489 token: Avx2,
11490 a: mask16x16<Avx2>,
11491 b: mask16x16<Avx2>,
11492 c: mask16x16<Avx2>,
11493 ) -> mask16x16<Avx2> {
11494 _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
11495 }
11496 );
11497 kernel(self, a, b, c)
11498 }
11499 #[inline(always)]
11500 fn simd_eq_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
11501 crate::kernel!(
11502 #[inline(always)]
11503 fn kernel(token: Avx2, a: mask16x16<Avx2>, b: mask16x16<Avx2>) -> mask16x16<Avx2> {
11504 _mm256_cmpeq_epi16(a.into(), b.into()).simd_into(token)
11505 }
11506 );
11507 kernel(self, a, b)
11508 }
11509 #[inline(always)]
11510 fn any_true_mask16x16(self, a: mask16x16<Self>) -> bool {
11511 crate::kernel!(
11512 #[inline(always)]
11513 fn kernel(token: Avx2, a: mask16x16<Avx2>) -> bool {
11514 _mm256_movemask_epi8(a.into()) as u32 != 0
11515 }
11516 );
11517 kernel(self, a)
11518 }
11519 #[inline(always)]
11520 fn all_true_mask16x16(self, a: mask16x16<Self>) -> bool {
11521 crate::kernel!(
11522 #[inline(always)]
11523 fn kernel(token: Avx2, a: mask16x16<Avx2>) -> bool {
11524 _mm256_movemask_epi8(a.into()) as u32 == 0xffffffff
11525 }
11526 );
11527 kernel(self, a)
11528 }
11529 #[inline(always)]
11530 fn any_false_mask16x16(self, a: mask16x16<Self>) -> bool {
11531 crate::kernel!(
11532 #[inline(always)]
11533 fn kernel(token: Avx2, a: mask16x16<Avx2>) -> bool {
11534 _mm256_movemask_epi8(a.into()) as u32 != 0xffffffff
11535 }
11536 );
11537 kernel(self, a)
11538 }
11539 #[inline(always)]
11540 fn all_false_mask16x16(self, a: mask16x16<Self>) -> bool {
11541 crate::kernel!(
11542 #[inline(always)]
11543 fn kernel(token: Avx2, a: mask16x16<Avx2>) -> bool {
11544 _mm256_movemask_epi8(a.into()) as u32 == 0
11545 }
11546 );
11547 kernel(self, a)
11548 }
11549 #[inline(always)]
11550 fn combine_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x32<Self> {
11551 mask16x32 {
11552 val: crate::support::Aligned512([a.val.0, b.val.0]),
11553 simd: self,
11554 }
11555 }
11556 #[inline(always)]
11557 fn split_mask16x16(self, a: mask16x16<Self>) -> (mask16x8<Self>, mask16x8<Self>) {
11558 crate::kernel!(
11559 #[inline(always)]
11560 fn kernel(token: Avx2, a: mask16x16<Avx2>) -> (mask16x8<Avx2>, mask16x8<Avx2>) {
11561 (
11562 _mm256_extracti128_si256::<0>(a.into()).simd_into(token),
11563 _mm256_extracti128_si256::<1>(a.into()).simd_into(token),
11564 )
11565 }
11566 );
11567 kernel(self, a)
11568 }
11569 #[inline(always)]
11570 fn widen_mask16x16(self, a: mask16x16<Self>) -> (mask32x8<Self>, mask32x8<Self>) {
11571 crate::kernel!(
11572 #[inline(always)]
11573 fn kernel(token: Avx2, a: mask16x16<Avx2>) -> (mask32x8<Avx2>, mask32x8<Avx2>) {
11574 let raw = a.into();
11575 (
11576 _mm256_cvtepi16_epi32(_mm256_castsi256_si128(raw)).simd_into(token),
11577 _mm256_cvtepi16_epi32(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
11578 )
11579 }
11580 );
11581 kernel(self, a)
11582 }
11583 #[inline(always)]
11584 fn narrow_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask8x32<Self> {
11585 crate::kernel!(
11586 #[inline(always)]
11587 fn kernel(token: Avx2, a: mask16x16<Avx2>, b: mask16x16<Avx2>) -> mask8x32<Avx2> {
11588 _mm256_permute4x64_epi64::<0xd8>(_mm256_packs_epi16(a.into(), b.into()))
11589 .simd_into(token)
11590 }
11591 );
11592 kernel(self, a, b)
11593 }
11594 #[inline(always)]
11595 fn abs_i32x8(self, a: i32x8<Self>) -> i32x8<Self> {
11596 crate::kernel!(
11597 #[inline(always)]
11598 fn kernel(token: Avx2, a: i32x8<Avx2>) -> i32x8<Avx2> {
11599 _mm256_abs_epi32(a.into()).simd_into(token)
11600 }
11601 );
11602 kernel(self, a)
11603 }
11604 #[inline(always)]
11605 fn splat_i32x8(self, val: i32) -> i32x8<Self> {
11606 crate::kernel!(
11607 #[inline(always)]
11608 fn kernel(token: Avx2, val: i32) -> i32x8<Avx2> {
11609 _mm256_set1_epi32(val).simd_into(token)
11610 }
11611 );
11612 kernel(self, val)
11613 }
11614 #[inline(always)]
11615 fn reverse_i32x8(self, a: i32x8<Self>) -> i32x8<Self> {
11616 let indices: u8x32<Self> = [
11617 28, 29, 30, 31, 24, 25, 26, 27, 20, 21, 22, 23, 16, 17, 18, 19, 12, 13, 14, 15, 8, 9,
11618 10, 11, 4, 5, 6, 7, 0, 1, 2, 3,
11619 ]
11620 .simd_into(self);
11621 self.swizzle_dyn_i32x8(a, indices)
11622 }
11623 #[inline(always)]
11624 fn slide_i32x8<const SHIFT: usize>(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11625 if SHIFT >= 8usize {
11626 return b;
11627 }
11628 let result = cross_block_alignr_256x1(
11629 self,
11630 Bytes::to_bytes(b).val.0,
11631 Bytes::to_bytes(a).val.0,
11632 SHIFT * 4usize,
11633 );
11634 Bytes::from_bytes(u8x32 {
11635 val: crate::support::Aligned256(result),
11636 simd: self,
11637 })
11638 }
11639 #[inline(always)]
11640 fn slide_within_blocks_i32x8<const SHIFT: usize>(
11641 self,
11642 a: i32x8<Self>,
11643 b: i32x8<Self>,
11644 ) -> i32x8<Self> {
11645 if SHIFT >= 4usize {
11646 return b;
11647 }
11648 let result = dyn_alignr_256(
11649 self,
11650 Bytes::to_bytes(b).val.0,
11651 Bytes::to_bytes(a).val.0,
11652 SHIFT * 4usize,
11653 );
11654 Bytes::from_bytes(u8x32 {
11655 val: crate::support::Aligned256(result),
11656 simd: self,
11657 })
11658 }
11659 #[inline(always)]
11660 fn compress_i32x8(self, values: i32x8<Self>, mask: mask32x8<Self>) -> i32x8<Self> {
11661 crate::kernel!(
11662 #[inline(always)]
11663 fn kernel(token: Avx2, values: i32x8<Avx2>, mask: mask32x8<Avx2>) -> i32x8<Avx2> {
11664 let bits = token.to_bitmask_mask32x8(mask) as usize & 255;
11665 let control: __m256i = crate::transmute::checked_transmute_copy(
11666 &crate::support::compact_256::COMPRESS_32[bits],
11667 );
11668 let reordered = _mm256_permutevar8x32_epi32(values.into(), control);
11669 let result = _mm256_andnot_si256(_mm256_srai_epi32::<31>(control), reordered);
11670 result.simd_into(token)
11671 }
11672 );
11673 kernel(self, values, mask)
11674 }
11675 #[inline(always)]
11676 fn compress_merge_i32x8(
11677 self,
11678 values: i32x8<Self>,
11679 mask: mask32x8<Self>,
11680 merge: i32x8<Self>,
11681 ) -> i32x8<Self> {
11682 crate::kernel!(
11683 #[inline(always)]
11684 fn kernel(
11685 token: Avx2,
11686 values: i32x8<Avx2>,
11687 mask: mask32x8<Avx2>,
11688 merge: i32x8<Avx2>,
11689 ) -> i32x8<Avx2> {
11690 let bits = token.to_bitmask_mask32x8(mask) as usize & 255;
11691 let control: __m256i = crate::transmute::checked_transmute_copy(
11692 &crate::support::compact_256::COMPRESS_32[bits],
11693 );
11694 let reordered = _mm256_permutevar8x32_epi32(values.into(), control);
11695 let result = _mm256_blendv_epi8(reordered, merge.into(), control);
11696 result.simd_into(token)
11697 }
11698 );
11699 kernel(self, values, mask, merge)
11700 }
11701 #[inline(always)]
11702 fn expand_i32x8(self, values: i32x8<Self>, mask: mask32x8<Self>) -> i32x8<Self> {
11703 crate::kernel!(
11704 #[inline(always)]
11705 fn kernel(token: Avx2, values: i32x8<Avx2>, mask: mask32x8<Avx2>) -> i32x8<Avx2> {
11706 let bits = token.to_bitmask_mask32x8(mask) as usize & 255;
11707 let control: __m256i = crate::transmute::checked_transmute_copy(
11708 &crate::support::compact_256::EXPAND_32[bits],
11709 );
11710 let reordered = _mm256_permutevar8x32_epi32(values.into(), control);
11711 let result = _mm256_and_si256(reordered, mask.into());
11712 result.simd_into(token)
11713 }
11714 );
11715 kernel(self, values, mask)
11716 }
11717 #[inline(always)]
11718 fn expand_merge_i32x8(
11719 self,
11720 values: i32x8<Self>,
11721 mask: mask32x8<Self>,
11722 merge: i32x8<Self>,
11723 ) -> i32x8<Self> {
11724 crate::kernel!(
11725 #[inline(always)]
11726 fn kernel(
11727 token: Avx2,
11728 values: i32x8<Avx2>,
11729 mask: mask32x8<Avx2>,
11730 merge: i32x8<Avx2>,
11731 ) -> i32x8<Avx2> {
11732 let bits = token.to_bitmask_mask32x8(mask) as usize & 255;
11733 let control: __m256i = crate::transmute::checked_transmute_copy(
11734 &crate::support::compact_256::EXPAND_32[bits],
11735 );
11736 let reordered = _mm256_permutevar8x32_epi32(values.into(), control);
11737 let result = _mm256_blendv_epi8(merge.into(), reordered, mask.into());
11738 result.simd_into(token)
11739 }
11740 );
11741 kernel(self, values, mask, merge)
11742 }
11743 #[inline(always)]
11744 fn count_ones_i32x8(self, a: i32x8<Self>) -> i32x8<Self> {
11745 crate::kernel!(
11746 #[inline(always)]
11747 fn kernel(token: Avx2, a: i32x8<Avx2>) -> i32x8<Avx2> {
11748 let value = a.into();
11749 let nibble_mask = _mm256_set1_epi8(0x0f);
11750 let lookup = _mm256_setr_epi8(
11751 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 0, 1, 1, 2, 1, 2, 2, 3, 1, 2,
11752 2, 3, 2, 3, 3, 4,
11753 );
11754 let low = _mm256_and_si256(value, nibble_mask);
11755 let high = _mm256_and_si256(_mm256_srli_epi16::<4>(value), nibble_mask);
11756 let byte_counts = _mm256_add_epi8(
11757 _mm256_shuffle_epi8(lookup, low),
11758 _mm256_shuffle_epi8(lookup, high),
11759 );
11760 _mm256_madd_epi16(
11761 _mm256_maddubs_epi16(byte_counts, _mm256_set1_epi8(1)),
11762 _mm256_set1_epi16(1),
11763 )
11764 .simd_into(token)
11765 }
11766 );
11767 kernel(self, a)
11768 }
11769 #[inline(always)]
11770 fn count_zeros_i32x8(self, a: i32x8<Self>) -> i32x8<Self> {
11771 self.count_ones_i32x8(self.not_i32x8(a))
11772 }
11773 #[inline(always)]
11774 fn add_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11775 crate::kernel!(
11776 #[inline(always)]
11777 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
11778 _mm256_add_epi32(a.into(), b.into()).simd_into(token)
11779 }
11780 );
11781 kernel(self, a, b)
11782 }
11783 #[inline(always)]
11784 fn saturating_add_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11785 crate::kernel!(
11786 #[inline(always)]
11787 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
11788 let a = a.into();
11789 let b = b.into();
11790 let wrapped = _mm256_add_epi32(a, b);
11791 let overflow = _mm256_xor_si256(_mm256_cmpgt_epi32(a, wrapped), b);
11792 let bound =
11793 _mm256_xor_si256(_mm256_srai_epi32::<31>(a), _mm256_set1_epi32(i32::MAX));
11794 let result = _mm256_blendv_ps(
11795 _mm256_castsi256_ps(wrapped),
11796 _mm256_castsi256_ps(bound),
11797 _mm256_castsi256_ps(overflow),
11798 );
11799 _mm256_castps_si256(result).simd_into(token)
11800 }
11801 );
11802 kernel(self, a, b)
11803 }
11804 #[inline(always)]
11805 fn sub_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11806 crate::kernel!(
11807 #[inline(always)]
11808 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
11809 _mm256_sub_epi32(a.into(), b.into()).simd_into(token)
11810 }
11811 );
11812 kernel(self, a, b)
11813 }
11814 #[inline(always)]
11815 fn saturating_sub_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11816 crate::kernel!(
11817 #[inline(always)]
11818 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
11819 let a = a.into();
11820 let b = b.into();
11821 let wrapped = _mm256_sub_epi32(a, b);
11822 let overflow = _mm256_xor_si256(_mm256_cmpgt_epi32(wrapped, a), b);
11823 let bound =
11824 _mm256_xor_si256(_mm256_srai_epi32::<31>(a), _mm256_set1_epi32(i32::MAX));
11825 let result = _mm256_blendv_ps(
11826 _mm256_castsi256_ps(wrapped),
11827 _mm256_castsi256_ps(bound),
11828 _mm256_castsi256_ps(overflow),
11829 );
11830 _mm256_castps_si256(result).simd_into(token)
11831 }
11832 );
11833 kernel(self, a, b)
11834 }
11835 #[inline(always)]
11836 fn mul_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11837 crate::kernel!(
11838 #[inline(always)]
11839 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
11840 _mm256_mullo_epi32(a.into(), b.into()).simd_into(token)
11841 }
11842 );
11843 kernel(self, a, b)
11844 }
11845 #[inline(always)]
11846 fn and_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11847 crate::kernel!(
11848 #[inline(always)]
11849 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
11850 _mm256_and_si256(a.into(), b.into()).simd_into(token)
11851 }
11852 );
11853 kernel(self, a, b)
11854 }
11855 #[inline(always)]
11856 fn or_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11857 crate::kernel!(
11858 #[inline(always)]
11859 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
11860 _mm256_or_si256(a.into(), b.into()).simd_into(token)
11861 }
11862 );
11863 kernel(self, a, b)
11864 }
11865 #[inline(always)]
11866 fn xor_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11867 crate::kernel!(
11868 #[inline(always)]
11869 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
11870 _mm256_xor_si256(a.into(), b.into()).simd_into(token)
11871 }
11872 );
11873 kernel(self, a, b)
11874 }
11875 #[inline(always)]
11876 fn not_i32x8(self, a: i32x8<Self>) -> i32x8<Self> {
11877 a ^ !0
11878 }
11879 #[inline(always)]
11880 fn shl_i32x8(self, a: i32x8<Self>, shift: u32) -> i32x8<Self> {
11881 crate::kernel!(
11882 #[inline(always)]
11883 fn kernel(token: Avx2, a: i32x8<Avx2>, shift: u32) -> i32x8<Avx2> {
11884 _mm256_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
11885 }
11886 );
11887 kernel(self, a, shift)
11888 }
11889 #[inline(always)]
11890 fn shlv_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11891 crate::kernel!(
11892 #[inline(always)]
11893 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
11894 _mm256_sllv_epi32(a.into(), b.into()).simd_into(token)
11895 }
11896 );
11897 kernel(self, a, b)
11898 }
11899 #[inline(always)]
11900 fn shr_i32x8(self, a: i32x8<Self>, shift: u32) -> i32x8<Self> {
11901 crate::kernel!(
11902 #[inline(always)]
11903 fn kernel(token: Avx2, a: i32x8<Avx2>, shift: u32) -> i32x8<Avx2> {
11904 _mm256_sra_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
11905 }
11906 );
11907 kernel(self, a, shift)
11908 }
11909 #[inline(always)]
11910 fn shrv_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11911 crate::kernel!(
11912 #[inline(always)]
11913 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
11914 _mm256_srav_epi32(a.into(), b.into()).simd_into(token)
11915 }
11916 );
11917 kernel(self, a, b)
11918 }
11919 #[inline(always)]
11920 fn max_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11921 crate::kernel!(
11922 #[inline(always)]
11923 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
11924 _mm256_max_epi32(a.into(), b.into()).simd_into(token)
11925 }
11926 );
11927 kernel(self, a, b)
11928 }
11929 #[inline(always)]
11930 fn min_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11931 crate::kernel!(
11932 #[inline(always)]
11933 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
11934 _mm256_min_epi32(a.into(), b.into()).simd_into(token)
11935 }
11936 );
11937 kernel(self, a, b)
11938 }
11939 #[inline(always)]
11940 fn clamp_i32x8(self, a: i32x8<Self>, b: i32x8<Self>, c: i32x8<Self>) -> i32x8<Self> {
11941 debug_assert!(
11942 b.simd_le(c).all_true(),
11943 "clamp requires min <= max for all elements"
11944 );
11945 a.max(b).min(c)
11946 }
11947 #[inline(always)]
11948 fn simd_eq_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> mask32x8<Self> {
11949 crate::kernel!(
11950 #[inline(always)]
11951 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> mask32x8<Avx2> {
11952 _mm256_cmpeq_epi32(a.into(), b.into()).simd_into(token)
11953 }
11954 );
11955 kernel(self, a, b)
11956 }
11957 #[inline(always)]
11958 fn simd_lt_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> mask32x8<Self> {
11959 crate::kernel!(
11960 #[inline(always)]
11961 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> mask32x8<Avx2> {
11962 _mm256_cmpgt_epi32(b.into(), a.into()).simd_into(token)
11963 }
11964 );
11965 kernel(self, a, b)
11966 }
11967 #[inline(always)]
11968 fn simd_le_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> mask32x8<Self> {
11969 crate::kernel!(
11970 #[inline(always)]
11971 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> mask32x8<Avx2> {
11972 _mm256_cmpeq_epi32(_mm256_min_epi32(a.into(), b.into()), a.into()).simd_into(token)
11973 }
11974 );
11975 kernel(self, a, b)
11976 }
11977 #[inline(always)]
11978 fn zip_low_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11979 crate::kernel!(
11980 #[inline(always)]
11981 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
11982 let lo = _mm256_unpacklo_epi32(a.into(), b.into());
11983 let hi = _mm256_unpackhi_epi32(a.into(), b.into());
11984 _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token)
11985 }
11986 );
11987 kernel(self, a, b)
11988 }
11989 #[inline(always)]
11990 fn zip_high_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
11991 crate::kernel!(
11992 #[inline(always)]
11993 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
11994 let lo = _mm256_unpacklo_epi32(a.into(), b.into());
11995 let hi = _mm256_unpackhi_epi32(a.into(), b.into());
11996 _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token)
11997 }
11998 );
11999 kernel(self, a, b)
12000 }
12001 #[inline(always)]
12002 fn unzip_low_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
12003 crate::kernel!(
12004 #[inline(always)]
12005 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
12006 let t1 = _mm256_permutevar8x32_epi32(
12007 a.into(),
12008 _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
12009 );
12010 let t2 = _mm256_permutevar8x32_epi32(
12011 b.into(),
12012 _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
12013 );
12014 _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token)
12015 }
12016 );
12017 kernel(self, a, b)
12018 }
12019 #[inline(always)]
12020 fn unzip_high_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
12021 crate::kernel!(
12022 #[inline(always)]
12023 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
12024 let t1 = _mm256_permutevar8x32_epi32(
12025 a.into(),
12026 _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
12027 );
12028 let t2 = _mm256_permutevar8x32_epi32(
12029 b.into(),
12030 _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
12031 );
12032 _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token)
12033 }
12034 );
12035 kernel(self, a, b)
12036 }
12037 #[inline(always)]
12038 fn interleave_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> (i32x8<Self>, i32x8<Self>) {
12039 crate::kernel!(
12040 #[inline(always)]
12041 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> (i32x8<Avx2>, i32x8<Avx2>) {
12042 let lo = _mm256_unpacklo_epi32(a.into(), b.into());
12043 let hi = _mm256_unpackhi_epi32(a.into(), b.into());
12044 (
12045 _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token),
12046 _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token),
12047 )
12048 }
12049 );
12050 kernel(self, a, b)
12051 }
12052 #[inline(always)]
12053 fn deinterleave_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> (i32x8<Self>, i32x8<Self>) {
12054 crate::kernel!(
12055 #[inline(always)]
12056 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> (i32x8<Avx2>, i32x8<Avx2>) {
12057 let t1 = _mm256_permutevar8x32_epi32(
12058 a.into(),
12059 _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
12060 );
12061 let t2 = _mm256_permutevar8x32_epi32(
12062 b.into(),
12063 _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
12064 );
12065 (
12066 _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token),
12067 _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token),
12068 )
12069 }
12070 );
12071 kernel(self, a, b)
12072 }
12073 #[inline(always)]
12074 fn select_i32x8(self, a: mask32x8<Self>, b: i32x8<Self>, c: i32x8<Self>) -> i32x8<Self> {
12075 crate::kernel!(
12076 #[inline(always)]
12077 fn kernel(
12078 token: Avx2,
12079 a: mask32x8<Avx2>,
12080 b: i32x8<Avx2>,
12081 c: i32x8<Avx2>,
12082 ) -> i32x8<Avx2> {
12083 _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
12084 }
12085 );
12086 kernel(self, a, b, c)
12087 }
12088 #[inline(always)]
12089 fn combine_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x16<Self> {
12090 i32x16 {
12091 val: crate::support::Aligned512([a.val.0, b.val.0]),
12092 simd: self,
12093 }
12094 }
12095 #[inline(always)]
12096 fn split_i32x8(self, a: i32x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
12097 crate::kernel!(
12098 #[inline(always)]
12099 fn kernel(token: Avx2, a: i32x8<Avx2>) -> (i32x4<Avx2>, i32x4<Avx2>) {
12100 (
12101 _mm256_extracti128_si256::<0>(a.into()).simd_into(token),
12102 _mm256_extracti128_si256::<1>(a.into()).simd_into(token),
12103 )
12104 }
12105 );
12106 kernel(self, a)
12107 }
12108 #[inline(always)]
12109 fn neg_i32x8(self, a: i32x8<Self>) -> i32x8<Self> {
12110 crate::kernel!(
12111 #[inline(always)]
12112 fn kernel(token: Avx2, a: i32x8<Avx2>) -> i32x8<Avx2> {
12113 _mm256_sub_epi32(_mm256_setzero_si256(), a.into()).simd_into(token)
12114 }
12115 );
12116 kernel(self, a)
12117 }
12118 #[inline(always)]
12119 fn widen_i32x8(self, a: i32x8<Self>) -> (i64x4<Self>, i64x4<Self>) {
12120 crate::kernel!(
12121 #[inline(always)]
12122 fn kernel(token: Avx2, a: i32x8<Avx2>) -> (i64x4<Avx2>, i64x4<Avx2>) {
12123 let raw = a.into();
12124 (
12125 _mm256_cvtepi32_epi64(_mm256_castsi256_si128(raw)).simd_into(token),
12126 _mm256_cvtepi32_epi64(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
12127 )
12128 }
12129 );
12130 kernel(self, a)
12131 }
12132 #[inline(always)]
12133 fn widening_mul_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> (i64x4<Self>, i64x4<Self>) {
12134 crate::kernel!(
12135 #[inline(always)]
12136 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> (i64x4<Avx2>, i64x4<Avx2>) {
12137 let a = a.into();
12138 let b = b.into();
12139 let low = _mm256_mul_epi32(a, b);
12140 let high = _mm256_mul_epi32(
12141 _mm256_shuffle_epi32::<0xf5>(a),
12142 _mm256_shuffle_epi32::<0xf5>(b),
12143 );
12144 let low = _mm256_permute4x64_epi64::<0xd8>(low);
12145 let high = _mm256_permute4x64_epi64::<0xd8>(high);
12146 let result_low = _mm256_unpacklo_epi64(low, high);
12147 let result_high = _mm256_unpackhi_epi64(low, high);
12148 (result_low.simd_into(token), result_high.simd_into(token))
12149 }
12150 );
12151 kernel(self, a, b)
12152 }
12153 #[inline(always)]
12154 fn widening_add_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> (i64x4<Self>, i64x4<Self>) {
12155 let (a0, a1) = self.widen_i32x8(a);
12156 let (b0, b1) = self.widen_i32x8(b);
12157 (self.add_i64x4(a0, b0), self.add_i64x4(a1, b1))
12158 }
12159 #[inline(always)]
12160 fn pairwise_widening_add_i32x8(self, a: i32x8<Self>) -> i64x4<Self> {
12161 crate::kernel!(
12162 #[inline(always)]
12163 fn kernel(token: Avx2, a: i32x8<Avx2>) -> i64x4<Avx2> {
12164 ({
12165 let y = _mm256_xor_si256(a.into(), _mm256_set1_epi32(i32::MIN));
12166 _mm256_add_epi64(
12167 _mm256_or_si256(y, _mm256_set1_epi64x(-0x1_0000_0000)),
12168 _mm256_srli_epi64::<32>(y),
12169 )
12170 })
12171 .simd_into(token)
12172 }
12173 );
12174 kernel(self, a)
12175 }
12176 #[inline(always)]
12177 fn pairwise_dot_product_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i64x4<Self> {
12178 let (low, high) = self.widening_mul_i32x8(a, b);
12179 let (even, odd) = self.deinterleave_i64x4(low, high);
12180 self.add_i64x4(even, odd)
12181 }
12182 #[inline(always)]
12183 fn narrow_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i16x16<Self> {
12184 crate::kernel!(
12185 #[inline(always)]
12186 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i16x16<Avx2> {
12187 _mm256_permute4x64_epi64::<0xd8>({
12188 let mask = _mm256_set1_epi32(0xffff);
12189 _mm256_packus_epi32(
12190 _mm256_and_si256(a.into(), mask),
12191 _mm256_and_si256(b.into(), mask),
12192 )
12193 })
12194 .simd_into(token)
12195 }
12196 );
12197 kernel(self, a, b)
12198 }
12199 #[inline(always)]
12200 fn saturating_narrow_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i16x16<Self> {
12201 crate::kernel!(
12202 #[inline(always)]
12203 fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i16x16<Avx2> {
12204 _mm256_permute4x64_epi64::<0xd8>(_mm256_packs_epi32(a.into(), b.into()))
12205 .simd_into(token)
12206 }
12207 );
12208 kernel(self, a, b)
12209 }
12210 #[inline(always)]
12211 fn relaxed_narrow_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i16x16<Self> {
12212 debug_assert!(
12213 a.as_slice()
12214 .iter()
12215 .chain(b.as_slice())
12216 .all(|&value| { value >= i16::MIN as i32 && value <= i16::MAX as i32 }),
12217 "relaxed_narrow inputs must fit in the destination type",
12218 );
12219 self.saturating_narrow_i32x8(a, b)
12220 }
12221 #[inline(always)]
12222 fn cvt_f32_i32x8(self, a: i32x8<Self>) -> f32x8<Self> {
12223 crate::kernel!(
12224 #[inline(always)]
12225 fn kernel(token: Avx2, a: i32x8<Avx2>) -> f32x8<Avx2> {
12226 _mm256_cvtepi32_ps(a.into()).simd_into(token)
12227 }
12228 );
12229 kernel(self, a)
12230 }
12231 #[inline(always)]
12232 fn splat_u32x8(self, val: u32) -> u32x8<Self> {
12233 crate::kernel!(
12234 #[inline(always)]
12235 fn kernel(token: Avx2, val: u32) -> u32x8<Avx2> {
12236 _mm256_set1_epi32(val.cast_signed()).simd_into(token)
12237 }
12238 );
12239 kernel(self, val)
12240 }
12241 #[inline(always)]
12242 fn reverse_u32x8(self, a: u32x8<Self>) -> u32x8<Self> {
12243 let indices: u8x32<Self> = [
12244 28, 29, 30, 31, 24, 25, 26, 27, 20, 21, 22, 23, 16, 17, 18, 19, 12, 13, 14, 15, 8, 9,
12245 10, 11, 4, 5, 6, 7, 0, 1, 2, 3,
12246 ]
12247 .simd_into(self);
12248 self.swizzle_dyn_u32x8(a, indices)
12249 }
12250 #[inline(always)]
12251 fn slide_u32x8<const SHIFT: usize>(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12252 if SHIFT >= 8usize {
12253 return b;
12254 }
12255 let result = cross_block_alignr_256x1(
12256 self,
12257 Bytes::to_bytes(b).val.0,
12258 Bytes::to_bytes(a).val.0,
12259 SHIFT * 4usize,
12260 );
12261 Bytes::from_bytes(u8x32 {
12262 val: crate::support::Aligned256(result),
12263 simd: self,
12264 })
12265 }
12266 #[inline(always)]
12267 fn slide_within_blocks_u32x8<const SHIFT: usize>(
12268 self,
12269 a: u32x8<Self>,
12270 b: u32x8<Self>,
12271 ) -> u32x8<Self> {
12272 if SHIFT >= 4usize {
12273 return b;
12274 }
12275 let result = dyn_alignr_256(
12276 self,
12277 Bytes::to_bytes(b).val.0,
12278 Bytes::to_bytes(a).val.0,
12279 SHIFT * 4usize,
12280 );
12281 Bytes::from_bytes(u8x32 {
12282 val: crate::support::Aligned256(result),
12283 simd: self,
12284 })
12285 }
12286 #[inline(always)]
12287 fn compress_u32x8(self, values: u32x8<Self>, mask: mask32x8<Self>) -> u32x8<Self> {
12288 crate::kernel!(
12289 #[inline(always)]
12290 fn kernel(token: Avx2, values: u32x8<Avx2>, mask: mask32x8<Avx2>) -> u32x8<Avx2> {
12291 let bits = token.to_bitmask_mask32x8(mask) as usize & 255;
12292 let control: __m256i = crate::transmute::checked_transmute_copy(
12293 &crate::support::compact_256::COMPRESS_32[bits],
12294 );
12295 let reordered = _mm256_permutevar8x32_epi32(values.into(), control);
12296 let result = _mm256_andnot_si256(_mm256_srai_epi32::<31>(control), reordered);
12297 result.simd_into(token)
12298 }
12299 );
12300 kernel(self, values, mask)
12301 }
12302 #[inline(always)]
12303 fn compress_merge_u32x8(
12304 self,
12305 values: u32x8<Self>,
12306 mask: mask32x8<Self>,
12307 merge: u32x8<Self>,
12308 ) -> u32x8<Self> {
12309 crate::kernel!(
12310 #[inline(always)]
12311 fn kernel(
12312 token: Avx2,
12313 values: u32x8<Avx2>,
12314 mask: mask32x8<Avx2>,
12315 merge: u32x8<Avx2>,
12316 ) -> u32x8<Avx2> {
12317 let bits = token.to_bitmask_mask32x8(mask) as usize & 255;
12318 let control: __m256i = crate::transmute::checked_transmute_copy(
12319 &crate::support::compact_256::COMPRESS_32[bits],
12320 );
12321 let reordered = _mm256_permutevar8x32_epi32(values.into(), control);
12322 let result = _mm256_blendv_epi8(reordered, merge.into(), control);
12323 result.simd_into(token)
12324 }
12325 );
12326 kernel(self, values, mask, merge)
12327 }
12328 #[inline(always)]
12329 fn expand_u32x8(self, values: u32x8<Self>, mask: mask32x8<Self>) -> u32x8<Self> {
12330 crate::kernel!(
12331 #[inline(always)]
12332 fn kernel(token: Avx2, values: u32x8<Avx2>, mask: mask32x8<Avx2>) -> u32x8<Avx2> {
12333 let bits = token.to_bitmask_mask32x8(mask) as usize & 255;
12334 let control: __m256i = crate::transmute::checked_transmute_copy(
12335 &crate::support::compact_256::EXPAND_32[bits],
12336 );
12337 let reordered = _mm256_permutevar8x32_epi32(values.into(), control);
12338 let result = _mm256_and_si256(reordered, mask.into());
12339 result.simd_into(token)
12340 }
12341 );
12342 kernel(self, values, mask)
12343 }
12344 #[inline(always)]
12345 fn expand_merge_u32x8(
12346 self,
12347 values: u32x8<Self>,
12348 mask: mask32x8<Self>,
12349 merge: u32x8<Self>,
12350 ) -> u32x8<Self> {
12351 crate::kernel!(
12352 #[inline(always)]
12353 fn kernel(
12354 token: Avx2,
12355 values: u32x8<Avx2>,
12356 mask: mask32x8<Avx2>,
12357 merge: u32x8<Avx2>,
12358 ) -> u32x8<Avx2> {
12359 let bits = token.to_bitmask_mask32x8(mask) as usize & 255;
12360 let control: __m256i = crate::transmute::checked_transmute_copy(
12361 &crate::support::compact_256::EXPAND_32[bits],
12362 );
12363 let reordered = _mm256_permutevar8x32_epi32(values.into(), control);
12364 let result = _mm256_blendv_epi8(merge.into(), reordered, mask.into());
12365 result.simd_into(token)
12366 }
12367 );
12368 kernel(self, values, mask, merge)
12369 }
12370 #[inline(always)]
12371 fn count_ones_u32x8(self, a: u32x8<Self>) -> u32x8<Self> {
12372 crate::kernel!(
12373 #[inline(always)]
12374 fn kernel(token: Avx2, a: u32x8<Avx2>) -> u32x8<Avx2> {
12375 let value = a.into();
12376 let nibble_mask = _mm256_set1_epi8(0x0f);
12377 let lookup = _mm256_setr_epi8(
12378 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 0, 1, 1, 2, 1, 2, 2, 3, 1, 2,
12379 2, 3, 2, 3, 3, 4,
12380 );
12381 let low = _mm256_and_si256(value, nibble_mask);
12382 let high = _mm256_and_si256(_mm256_srli_epi16::<4>(value), nibble_mask);
12383 let byte_counts = _mm256_add_epi8(
12384 _mm256_shuffle_epi8(lookup, low),
12385 _mm256_shuffle_epi8(lookup, high),
12386 );
12387 _mm256_madd_epi16(
12388 _mm256_maddubs_epi16(byte_counts, _mm256_set1_epi8(1)),
12389 _mm256_set1_epi16(1),
12390 )
12391 .simd_into(token)
12392 }
12393 );
12394 kernel(self, a)
12395 }
12396 #[inline(always)]
12397 fn count_zeros_u32x8(self, a: u32x8<Self>) -> u32x8<Self> {
12398 self.count_ones_u32x8(self.not_u32x8(a))
12399 }
12400 #[inline(always)]
12401 fn add_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12402 crate::kernel!(
12403 #[inline(always)]
12404 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
12405 _mm256_add_epi32(a.into(), b.into()).simd_into(token)
12406 }
12407 );
12408 kernel(self, a, b)
12409 }
12410 #[inline(always)]
12411 fn saturating_add_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12412 crate::kernel!(
12413 #[inline(always)]
12414 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
12415 let a = a.into();
12416 let b = b.into();
12417 let threshold = _mm256_xor_si256(b, _mm256_set1_epi32(-1));
12418 _mm256_add_epi32(_mm256_min_epu32(a, threshold), b).simd_into(token)
12419 }
12420 );
12421 kernel(self, a, b)
12422 }
12423 #[inline(always)]
12424 fn sub_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12425 crate::kernel!(
12426 #[inline(always)]
12427 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
12428 _mm256_sub_epi32(a.into(), b.into()).simd_into(token)
12429 }
12430 );
12431 kernel(self, a, b)
12432 }
12433 #[inline(always)]
12434 fn saturating_sub_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12435 crate::kernel!(
12436 #[inline(always)]
12437 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
12438 let a = a.into();
12439 let b = b.into();
12440 _mm256_sub_epi32(_mm256_max_epu32(a, b), b).simd_into(token)
12441 }
12442 );
12443 kernel(self, a, b)
12444 }
12445 #[inline(always)]
12446 fn mul_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12447 crate::kernel!(
12448 #[inline(always)]
12449 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
12450 _mm256_mullo_epi32(a.into(), b.into()).simd_into(token)
12451 }
12452 );
12453 kernel(self, a, b)
12454 }
12455 #[inline(always)]
12456 fn and_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12457 crate::kernel!(
12458 #[inline(always)]
12459 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
12460 _mm256_and_si256(a.into(), b.into()).simd_into(token)
12461 }
12462 );
12463 kernel(self, a, b)
12464 }
12465 #[inline(always)]
12466 fn or_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12467 crate::kernel!(
12468 #[inline(always)]
12469 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
12470 _mm256_or_si256(a.into(), b.into()).simd_into(token)
12471 }
12472 );
12473 kernel(self, a, b)
12474 }
12475 #[inline(always)]
12476 fn xor_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12477 crate::kernel!(
12478 #[inline(always)]
12479 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
12480 _mm256_xor_si256(a.into(), b.into()).simd_into(token)
12481 }
12482 );
12483 kernel(self, a, b)
12484 }
12485 #[inline(always)]
12486 fn not_u32x8(self, a: u32x8<Self>) -> u32x8<Self> {
12487 a ^ !0
12488 }
12489 #[inline(always)]
12490 fn shl_u32x8(self, a: u32x8<Self>, shift: u32) -> u32x8<Self> {
12491 crate::kernel!(
12492 #[inline(always)]
12493 fn kernel(token: Avx2, a: u32x8<Avx2>, shift: u32) -> u32x8<Avx2> {
12494 _mm256_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
12495 }
12496 );
12497 kernel(self, a, shift)
12498 }
12499 #[inline(always)]
12500 fn shlv_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12501 crate::kernel!(
12502 #[inline(always)]
12503 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
12504 _mm256_sllv_epi32(a.into(), b.into()).simd_into(token)
12505 }
12506 );
12507 kernel(self, a, b)
12508 }
12509 #[inline(always)]
12510 fn shr_u32x8(self, a: u32x8<Self>, shift: u32) -> u32x8<Self> {
12511 crate::kernel!(
12512 #[inline(always)]
12513 fn kernel(token: Avx2, a: u32x8<Avx2>, shift: u32) -> u32x8<Avx2> {
12514 _mm256_srl_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
12515 }
12516 );
12517 kernel(self, a, shift)
12518 }
12519 #[inline(always)]
12520 fn shrv_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12521 crate::kernel!(
12522 #[inline(always)]
12523 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
12524 _mm256_srlv_epi32(a.into(), b.into()).simd_into(token)
12525 }
12526 );
12527 kernel(self, a, b)
12528 }
12529 #[inline(always)]
12530 fn max_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12531 crate::kernel!(
12532 #[inline(always)]
12533 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
12534 _mm256_max_epu32(a.into(), b.into()).simd_into(token)
12535 }
12536 );
12537 kernel(self, a, b)
12538 }
12539 #[inline(always)]
12540 fn min_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12541 crate::kernel!(
12542 #[inline(always)]
12543 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
12544 _mm256_min_epu32(a.into(), b.into()).simd_into(token)
12545 }
12546 );
12547 kernel(self, a, b)
12548 }
12549 #[inline(always)]
12550 fn clamp_u32x8(self, a: u32x8<Self>, b: u32x8<Self>, c: u32x8<Self>) -> u32x8<Self> {
12551 debug_assert!(
12552 b.simd_le(c).all_true(),
12553 "clamp requires min <= max for all elements"
12554 );
12555 a.max(b).min(c)
12556 }
12557 #[inline(always)]
12558 fn simd_eq_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> mask32x8<Self> {
12559 crate::kernel!(
12560 #[inline(always)]
12561 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> mask32x8<Avx2> {
12562 _mm256_cmpeq_epi32(a.into(), b.into()).simd_into(token)
12563 }
12564 );
12565 kernel(self, a, b)
12566 }
12567 #[inline(always)]
12568 fn simd_lt_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> mask32x8<Self> {
12569 crate::kernel!(
12570 #[inline(always)]
12571 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> mask32x8<Avx2> {
12572 {
12573 let sign_bit = _mm256_set1_epi32(0x80000000u32.cast_signed());
12574 let lhs_signed = _mm256_xor_si256(b.into(), sign_bit);
12575 let rhs_signed = _mm256_xor_si256(a.into(), sign_bit);
12576 _mm256_cmpgt_epi32(lhs_signed, rhs_signed)
12577 }
12578 .simd_into(token)
12579 }
12580 );
12581 kernel(self, a, b)
12582 }
12583 #[inline(always)]
12584 fn simd_le_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> mask32x8<Self> {
12585 crate::kernel!(
12586 #[inline(always)]
12587 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> mask32x8<Avx2> {
12588 _mm256_cmpeq_epi32(_mm256_min_epu32(a.into(), b.into()), a.into()).simd_into(token)
12589 }
12590 );
12591 kernel(self, a, b)
12592 }
12593 #[inline(always)]
12594 fn zip_low_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12595 crate::kernel!(
12596 #[inline(always)]
12597 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
12598 let lo = _mm256_unpacklo_epi32(a.into(), b.into());
12599 let hi = _mm256_unpackhi_epi32(a.into(), b.into());
12600 _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token)
12601 }
12602 );
12603 kernel(self, a, b)
12604 }
12605 #[inline(always)]
12606 fn zip_high_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12607 crate::kernel!(
12608 #[inline(always)]
12609 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
12610 let lo = _mm256_unpacklo_epi32(a.into(), b.into());
12611 let hi = _mm256_unpackhi_epi32(a.into(), b.into());
12612 _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token)
12613 }
12614 );
12615 kernel(self, a, b)
12616 }
12617 #[inline(always)]
12618 fn unzip_low_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12619 crate::kernel!(
12620 #[inline(always)]
12621 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
12622 let t1 = _mm256_permutevar8x32_epi32(
12623 a.into(),
12624 _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
12625 );
12626 let t2 = _mm256_permutevar8x32_epi32(
12627 b.into(),
12628 _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
12629 );
12630 _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token)
12631 }
12632 );
12633 kernel(self, a, b)
12634 }
12635 #[inline(always)]
12636 fn unzip_high_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
12637 crate::kernel!(
12638 #[inline(always)]
12639 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
12640 let t1 = _mm256_permutevar8x32_epi32(
12641 a.into(),
12642 _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
12643 );
12644 let t2 = _mm256_permutevar8x32_epi32(
12645 b.into(),
12646 _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
12647 );
12648 _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token)
12649 }
12650 );
12651 kernel(self, a, b)
12652 }
12653 #[inline(always)]
12654 fn interleave_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> (u32x8<Self>, u32x8<Self>) {
12655 crate::kernel!(
12656 #[inline(always)]
12657 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> (u32x8<Avx2>, u32x8<Avx2>) {
12658 let lo = _mm256_unpacklo_epi32(a.into(), b.into());
12659 let hi = _mm256_unpackhi_epi32(a.into(), b.into());
12660 (
12661 _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token),
12662 _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token),
12663 )
12664 }
12665 );
12666 kernel(self, a, b)
12667 }
12668 #[inline(always)]
12669 fn deinterleave_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> (u32x8<Self>, u32x8<Self>) {
12670 crate::kernel!(
12671 #[inline(always)]
12672 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> (u32x8<Avx2>, u32x8<Avx2>) {
12673 let t1 = _mm256_permutevar8x32_epi32(
12674 a.into(),
12675 _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
12676 );
12677 let t2 = _mm256_permutevar8x32_epi32(
12678 b.into(),
12679 _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
12680 );
12681 (
12682 _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token),
12683 _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token),
12684 )
12685 }
12686 );
12687 kernel(self, a, b)
12688 }
12689 #[inline(always)]
12690 fn select_u32x8(self, a: mask32x8<Self>, b: u32x8<Self>, c: u32x8<Self>) -> u32x8<Self> {
12691 crate::kernel!(
12692 #[inline(always)]
12693 fn kernel(
12694 token: Avx2,
12695 a: mask32x8<Avx2>,
12696 b: u32x8<Avx2>,
12697 c: u32x8<Avx2>,
12698 ) -> u32x8<Avx2> {
12699 _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
12700 }
12701 );
12702 kernel(self, a, b, c)
12703 }
12704 #[inline(always)]
12705 fn combine_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x16<Self> {
12706 u32x16 {
12707 val: crate::support::Aligned512([a.val.0, b.val.0]),
12708 simd: self,
12709 }
12710 }
12711 #[inline(always)]
12712 fn split_u32x8(self, a: u32x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
12713 crate::kernel!(
12714 #[inline(always)]
12715 fn kernel(token: Avx2, a: u32x8<Avx2>) -> (u32x4<Avx2>, u32x4<Avx2>) {
12716 (
12717 _mm256_extracti128_si256::<0>(a.into()).simd_into(token),
12718 _mm256_extracti128_si256::<1>(a.into()).simd_into(token),
12719 )
12720 }
12721 );
12722 kernel(self, a)
12723 }
12724 #[inline(always)]
12725 fn widen_u32x8(self, a: u32x8<Self>) -> (u64x4<Self>, u64x4<Self>) {
12726 crate::kernel!(
12727 #[inline(always)]
12728 fn kernel(token: Avx2, a: u32x8<Avx2>) -> (u64x4<Avx2>, u64x4<Avx2>) {
12729 let raw = a.into();
12730 (
12731 _mm256_cvtepu32_epi64(_mm256_castsi256_si128(raw)).simd_into(token),
12732 _mm256_cvtepu32_epi64(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
12733 )
12734 }
12735 );
12736 kernel(self, a)
12737 }
12738 #[inline(always)]
12739 fn widening_mul_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> (u64x4<Self>, u64x4<Self>) {
12740 let (a0, a1) = self.widen_u32x8(a);
12741 let (b0, b1) = self.widen_u32x8(b);
12742 (self.mul_u64x4(a0, b0), self.mul_u64x4(a1, b1))
12743 }
12744 #[inline(always)]
12745 fn widening_add_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> (u64x4<Self>, u64x4<Self>) {
12746 let (a0, a1) = self.widen_u32x8(a);
12747 let (b0, b1) = self.widen_u32x8(b);
12748 (self.add_u64x4(a0, b0), self.add_u64x4(a1, b1))
12749 }
12750 #[inline(always)]
12751 fn pairwise_widening_add_u32x8(self, a: u32x8<Self>) -> u64x4<Self> {
12752 crate::kernel!(
12753 #[inline(always)]
12754 fn kernel(token: Avx2, a: u32x8<Avx2>) -> u64x4<Avx2> {
12755 ({
12756 let a = a.into();
12757 _mm256_add_epi64(
12758 _mm256_and_si256(a, _mm256_set1_epi64x(4294967295)),
12759 _mm256_srli_epi64::<32>(a),
12760 )
12761 })
12762 .simd_into(token)
12763 }
12764 );
12765 kernel(self, a)
12766 }
12767 #[inline(always)]
12768 fn pairwise_dot_product_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u64x4<Self> {
12769 let (low, high) = self.widening_mul_u32x8(a, b);
12770 let (even, odd) = self.deinterleave_u64x4(low, high);
12771 self.add_u64x4(even, odd)
12772 }
12773 #[inline(always)]
12774 fn narrow_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u16x16<Self> {
12775 crate::kernel!(
12776 #[inline(always)]
12777 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u16x16<Avx2> {
12778 _mm256_permute4x64_epi64::<0xd8>({
12779 let mask = _mm256_set1_epi32(0xffff);
12780 _mm256_packus_epi32(
12781 _mm256_and_si256(a.into(), mask),
12782 _mm256_and_si256(b.into(), mask),
12783 )
12784 })
12785 .simd_into(token)
12786 }
12787 );
12788 kernel(self, a, b)
12789 }
12790 #[inline(always)]
12791 fn saturating_narrow_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u16x16<Self> {
12792 crate::kernel!(
12793 #[inline(always)]
12794 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u16x16<Avx2> {
12795 _mm256_permute4x64_epi64::<0xd8>({
12796 let max = _mm256_set1_epi32(i32::MAX);
12797 _mm256_packus_epi32(
12798 _mm256_min_epu32(a.into(), max),
12799 _mm256_min_epu32(b.into(), max),
12800 )
12801 })
12802 .simd_into(token)
12803 }
12804 );
12805 kernel(self, a, b)
12806 }
12807 #[inline(always)]
12808 fn relaxed_narrow_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u16x16<Self> {
12809 crate::kernel!(
12810 #[inline(always)]
12811 fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u16x16<Avx2> {
12812 debug_assert!(
12813 a.as_slice()
12814 .iter()
12815 .chain(b.as_slice())
12816 .all(|&value| { value >= u16::MIN as u32 && value <= u16::MAX as u32 }),
12817 "relaxed_narrow inputs must fit in the destination type",
12818 );
12819 _mm256_permute4x64_epi64::<0xd8>(_mm256_packus_epi32(a.into(), b.into()))
12820 .simd_into(token)
12821 }
12822 );
12823 kernel(self, a, b)
12824 }
12825 #[inline(always)]
12826 fn cvt_f32_u32x8(self, a: u32x8<Self>) -> f32x8<Self> {
12827 crate::kernel!(
12828 #[inline(always)]
12829 fn kernel(token: Avx2, a: u32x8<Avx2>) -> f32x8<Avx2> {
12830 let a = a.into();
12831 let lo = _mm256_blend_epi16::<0xAA>(a, _mm256_set1_epi32(0x4B000000));
12832 let hi = _mm256_blend_epi16::<0xAA>(
12833 _mm256_srli_epi32::<16>(a),
12834 _mm256_set1_epi32(0x53000000),
12835 );
12836 let fhi = _mm256_sub_ps(
12837 _mm256_castsi256_ps(hi),
12838 _mm256_set1_ps(f32::from_bits(0x53000080)),
12839 );
12840 let result = _mm256_add_ps(_mm256_castsi256_ps(lo), fhi);
12841 result.simd_into(token)
12842 }
12843 );
12844 kernel(self, a)
12845 }
12846 #[inline(always)]
12847 fn splat_mask32x8(self, val: bool) -> mask32x8<Self> {
12848 crate::kernel!(
12849 #[inline(always)]
12850 fn kernel(token: Avx2, val: bool) -> mask32x8<Avx2> {
12851 let val: i32 = if val { !0 } else { 0 };
12852 _mm256_set1_epi32(val).simd_into(token)
12853 }
12854 );
12855 kernel(self, val)
12856 }
12857 #[inline(always)]
12858 fn from_bitmask_mask32x8(self, bits: u64) -> mask32x8<Self> {
12859 crate::kernel!(
12860 #[inline(always)]
12861 fn kernel(token: Avx2, bits: u64) -> mask32x8<Avx2> {
12862 {
12863 let bit_lanes = _mm256_set1_epi32(bits as i32);
12864 let bit_mask = _mm256_setr_epi32(1, 2, 4, 8, 16, 32, 64, 128);
12865 _mm256_cmpeq_epi32(_mm256_and_si256(bit_lanes, bit_mask), bit_mask)
12866 }
12867 .simd_into(token)
12868 }
12869 );
12870 kernel(self, bits)
12871 }
12872 #[inline(always)]
12873 fn to_bitmask_mask32x8(self, a: mask32x8<Self>) -> u64 {
12874 crate::kernel!(
12875 #[inline(always)]
12876 fn kernel(token: Avx2, a: mask32x8<Avx2>) -> u64 {
12877 _mm256_movemask_ps(_mm256_castsi256_ps(a.into())) as u32 as u64
12878 }
12879 );
12880 kernel(self, a)
12881 }
12882 #[inline(always)]
12883 fn from_vector_mask32x8(self, vector: i32x8<Self>) -> mask32x8<Self> {
12884 mask32x8 {
12885 val: vector.val,
12886 simd: self,
12887 }
12888 }
12889 #[inline(always)]
12890 fn to_vector_mask32x8(self, a: mask32x8<Self>) -> i32x8<Self> {
12891 i32x8 {
12892 val: a.val,
12893 simd: self,
12894 }
12895 }
12896 #[inline(always)]
12897 fn set_mask32x8(self, a: &mut mask32x8<Self>, index: usize, value: bool) -> () {
12898 assert!(
12899 index < 8usize,
12900 "mask lane index {index} is out of bounds for {} lanes",
12901 8usize
12902 );
12903 let mut lanes: [i32; 8usize] = (*a).into();
12904 lanes[index] = if value { !0 } else { 0 };
12905 *a = lanes.simd_into(self);
12906 }
12907 #[inline(always)]
12908 fn rotate_elements_left_mask32x8<const OFFSET: usize>(
12909 self,
12910 a: mask32x8<Self>,
12911 ) -> mask32x8<Self> {
12912 let int = i32x8 {
12913 val: crate::transmute::checked_transmute_copy(&a.val),
12914 simd: self,
12915 };
12916 let rotated = int.rotate_elements_left::<OFFSET>();
12917 mask32x8 {
12918 val: crate::transmute::checked_transmute_copy(&rotated.val),
12919 simd: self,
12920 }
12921 }
12922 #[inline(always)]
12923 fn rotate_elements_right_mask32x8<const OFFSET: usize>(
12924 self,
12925 a: mask32x8<Self>,
12926 ) -> mask32x8<Self> {
12927 let int = i32x8 {
12928 val: crate::transmute::checked_transmute_copy(&a.val),
12929 simd: self,
12930 };
12931 let rotated = int.rotate_elements_right::<OFFSET>();
12932 mask32x8 {
12933 val: crate::transmute::checked_transmute_copy(&rotated.val),
12934 simd: self,
12935 }
12936 }
12937 #[inline(always)]
12938 fn and_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
12939 crate::kernel!(
12940 #[inline(always)]
12941 fn kernel(token: Avx2, a: mask32x8<Avx2>, b: mask32x8<Avx2>) -> mask32x8<Avx2> {
12942 _mm256_and_si256(a.into(), b.into()).simd_into(token)
12943 }
12944 );
12945 kernel(self, a, b)
12946 }
12947 #[inline(always)]
12948 fn or_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
12949 crate::kernel!(
12950 #[inline(always)]
12951 fn kernel(token: Avx2, a: mask32x8<Avx2>, b: mask32x8<Avx2>) -> mask32x8<Avx2> {
12952 _mm256_or_si256(a.into(), b.into()).simd_into(token)
12953 }
12954 );
12955 kernel(self, a, b)
12956 }
12957 #[inline(always)]
12958 fn xor_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
12959 crate::kernel!(
12960 #[inline(always)]
12961 fn kernel(token: Avx2, a: mask32x8<Avx2>, b: mask32x8<Avx2>) -> mask32x8<Avx2> {
12962 _mm256_xor_si256(a.into(), b.into()).simd_into(token)
12963 }
12964 );
12965 kernel(self, a, b)
12966 }
12967 #[inline(always)]
12968 fn not_mask32x8(self, a: mask32x8<Self>) -> mask32x8<Self> {
12969 self.xor_mask32x8(a, self.splat_mask32x8(true))
12970 }
12971 #[inline(always)]
12972 fn reverse_mask32x8(self, a: mask32x8<Self>) -> mask32x8<Self> {
12973 let lanes = i32x8 {
12974 val: crate::transmute::checked_transmute_copy(&a.val),
12975 simd: self,
12976 };
12977 let reversed = self.reverse_i32x8(lanes);
12978 mask32x8 {
12979 val: crate::transmute::checked_transmute_copy(&reversed.val),
12980 simd: self,
12981 }
12982 }
12983 #[inline(always)]
12984 fn select_mask32x8(
12985 self,
12986 a: mask32x8<Self>,
12987 b: mask32x8<Self>,
12988 c: mask32x8<Self>,
12989 ) -> mask32x8<Self> {
12990 crate::kernel!(
12991 #[inline(always)]
12992 fn kernel(
12993 token: Avx2,
12994 a: mask32x8<Avx2>,
12995 b: mask32x8<Avx2>,
12996 c: mask32x8<Avx2>,
12997 ) -> mask32x8<Avx2> {
12998 _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
12999 }
13000 );
13001 kernel(self, a, b, c)
13002 }
13003 #[inline(always)]
13004 fn simd_eq_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
13005 crate::kernel!(
13006 #[inline(always)]
13007 fn kernel(token: Avx2, a: mask32x8<Avx2>, b: mask32x8<Avx2>) -> mask32x8<Avx2> {
13008 _mm256_cmpeq_epi32(a.into(), b.into()).simd_into(token)
13009 }
13010 );
13011 kernel(self, a, b)
13012 }
13013 #[inline(always)]
13014 fn any_true_mask32x8(self, a: mask32x8<Self>) -> bool {
13015 crate::kernel!(
13016 #[inline(always)]
13017 fn kernel(token: Avx2, a: mask32x8<Avx2>) -> bool {
13018 _mm256_movemask_ps(_mm256_castsi256_ps(a.into())) as u32 != 0
13019 }
13020 );
13021 kernel(self, a)
13022 }
13023 #[inline(always)]
13024 fn all_true_mask32x8(self, a: mask32x8<Self>) -> bool {
13025 crate::kernel!(
13026 #[inline(always)]
13027 fn kernel(token: Avx2, a: mask32x8<Avx2>) -> bool {
13028 _mm256_movemask_ps(_mm256_castsi256_ps(a.into())) as u32 == 0b11111111
13029 }
13030 );
13031 kernel(self, a)
13032 }
13033 #[inline(always)]
13034 fn any_false_mask32x8(self, a: mask32x8<Self>) -> bool {
13035 crate::kernel!(
13036 #[inline(always)]
13037 fn kernel(token: Avx2, a: mask32x8<Avx2>) -> bool {
13038 _mm256_movemask_ps(_mm256_castsi256_ps(a.into())) as u32 != 0b11111111
13039 }
13040 );
13041 kernel(self, a)
13042 }
13043 #[inline(always)]
13044 fn all_false_mask32x8(self, a: mask32x8<Self>) -> bool {
13045 crate::kernel!(
13046 #[inline(always)]
13047 fn kernel(token: Avx2, a: mask32x8<Avx2>) -> bool {
13048 _mm256_movemask_ps(_mm256_castsi256_ps(a.into())) as u32 == 0
13049 }
13050 );
13051 kernel(self, a)
13052 }
13053 #[inline(always)]
13054 fn combine_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x16<Self> {
13055 mask32x16 {
13056 val: crate::support::Aligned512([a.val.0, b.val.0]),
13057 simd: self,
13058 }
13059 }
13060 #[inline(always)]
13061 fn split_mask32x8(self, a: mask32x8<Self>) -> (mask32x4<Self>, mask32x4<Self>) {
13062 crate::kernel!(
13063 #[inline(always)]
13064 fn kernel(token: Avx2, a: mask32x8<Avx2>) -> (mask32x4<Avx2>, mask32x4<Avx2>) {
13065 (
13066 _mm256_extracti128_si256::<0>(a.into()).simd_into(token),
13067 _mm256_extracti128_si256::<1>(a.into()).simd_into(token),
13068 )
13069 }
13070 );
13071 kernel(self, a)
13072 }
13073 #[inline(always)]
13074 fn widen_mask32x8(self, a: mask32x8<Self>) -> (mask64x4<Self>, mask64x4<Self>) {
13075 crate::kernel!(
13076 #[inline(always)]
13077 fn kernel(token: Avx2, a: mask32x8<Avx2>) -> (mask64x4<Avx2>, mask64x4<Avx2>) {
13078 let raw = a.into();
13079 (
13080 _mm256_cvtepi32_epi64(_mm256_castsi256_si128(raw)).simd_into(token),
13081 _mm256_cvtepi32_epi64(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
13082 )
13083 }
13084 );
13085 kernel(self, a)
13086 }
13087 #[inline(always)]
13088 fn narrow_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask16x16<Self> {
13089 crate::kernel!(
13090 #[inline(always)]
13091 fn kernel(token: Avx2, a: mask32x8<Avx2>, b: mask32x8<Avx2>) -> mask16x16<Avx2> {
13092 _mm256_permute4x64_epi64::<0xd8>(_mm256_packs_epi32(a.into(), b.into()))
13093 .simd_into(token)
13094 }
13095 );
13096 kernel(self, a, b)
13097 }
13098 #[inline(always)]
13099 fn abs_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13100 crate::kernel!(
13101 #[inline(always)]
13102 fn kernel(token: Avx2, a: f64x4<Avx2>) -> f64x4<Avx2> {
13103 _mm256_andnot_pd(_mm256_set1_pd(-0.0), a.into()).simd_into(token)
13104 }
13105 );
13106 kernel(self, a)
13107 }
13108 #[inline(always)]
13109 fn splat_f64x4(self, val: f64) -> f64x4<Self> {
13110 crate::kernel!(
13111 #[inline(always)]
13112 fn kernel(token: Avx2, val: f64) -> f64x4<Avx2> {
13113 _mm256_set1_pd(val).simd_into(token)
13114 }
13115 );
13116 kernel(self, val)
13117 }
13118 #[inline(always)]
13119 fn reverse_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13120 let indices: u8x32<Self> = [
13121 24, 25, 26, 27, 28, 29, 30, 31, 16, 17, 18, 19, 20, 21, 22, 23, 8, 9, 10, 11, 12, 13,
13122 14, 15, 0, 1, 2, 3, 4, 5, 6, 7,
13123 ]
13124 .simd_into(self);
13125 self.swizzle_dyn_f64x4(a, indices)
13126 }
13127 #[inline(always)]
13128 fn slide_f64x4<const SHIFT: usize>(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13129 if SHIFT >= 4usize {
13130 return b;
13131 }
13132 let result = cross_block_alignr_256x1(
13133 self,
13134 Bytes::to_bytes(b).val.0,
13135 Bytes::to_bytes(a).val.0,
13136 SHIFT * 8usize,
13137 );
13138 Bytes::from_bytes(u8x32 {
13139 val: crate::support::Aligned256(result),
13140 simd: self,
13141 })
13142 }
13143 #[inline(always)]
13144 fn slide_within_blocks_f64x4<const SHIFT: usize>(
13145 self,
13146 a: f64x4<Self>,
13147 b: f64x4<Self>,
13148 ) -> f64x4<Self> {
13149 if SHIFT >= 2usize {
13150 return b;
13151 }
13152 let result = dyn_alignr_256(
13153 self,
13154 Bytes::to_bytes(b).val.0,
13155 Bytes::to_bytes(a).val.0,
13156 SHIFT * 8usize,
13157 );
13158 Bytes::from_bytes(u8x32 {
13159 val: crate::support::Aligned256(result),
13160 simd: self,
13161 })
13162 }
13163 #[inline(always)]
13164 fn compress_f64x4(self, values: f64x4<Self>, mask: mask64x4<Self>) -> f64x4<Self> {
13165 crate::kernel!(
13166 #[inline(always)]
13167 fn kernel(token: Avx2, values: f64x4<Avx2>, mask: mask64x4<Avx2>) -> f64x4<Avx2> {
13168 let bits = token.to_bitmask_mask64x4(mask) as usize & 15;
13169 let control: __m256i = crate::transmute::checked_transmute_copy(
13170 &crate::support::compact_256::COMPRESS_64[bits],
13171 );
13172 let reordered =
13173 _mm256_permutevar8x32_epi32(_mm256_castpd_si256(values.into()), control);
13174 let result = _mm256_andnot_si256(_mm256_srai_epi32::<31>(control), reordered);
13175 _mm256_castsi256_pd(result).simd_into(token)
13176 }
13177 );
13178 kernel(self, values, mask)
13179 }
13180 #[inline(always)]
13181 fn compress_merge_f64x4(
13182 self,
13183 values: f64x4<Self>,
13184 mask: mask64x4<Self>,
13185 merge: f64x4<Self>,
13186 ) -> f64x4<Self> {
13187 crate::kernel!(
13188 #[inline(always)]
13189 fn kernel(
13190 token: Avx2,
13191 values: f64x4<Avx2>,
13192 mask: mask64x4<Avx2>,
13193 merge: f64x4<Avx2>,
13194 ) -> f64x4<Avx2> {
13195 let bits = token.to_bitmask_mask64x4(mask) as usize & 15;
13196 let control: __m256i = crate::transmute::checked_transmute_copy(
13197 &crate::support::compact_256::COMPRESS_64[bits],
13198 );
13199 let reordered =
13200 _mm256_permutevar8x32_epi32(_mm256_castpd_si256(values.into()), control);
13201 let result =
13202 _mm256_blendv_epi8(reordered, _mm256_castpd_si256(merge.into()), control);
13203 _mm256_castsi256_pd(result).simd_into(token)
13204 }
13205 );
13206 kernel(self, values, mask, merge)
13207 }
13208 #[inline(always)]
13209 fn expand_f64x4(self, values: f64x4<Self>, mask: mask64x4<Self>) -> f64x4<Self> {
13210 crate::kernel!(
13211 #[inline(always)]
13212 fn kernel(token: Avx2, values: f64x4<Avx2>, mask: mask64x4<Avx2>) -> f64x4<Avx2> {
13213 let bits = token.to_bitmask_mask64x4(mask) as usize & 15;
13214 let control: __m256i = crate::transmute::checked_transmute_copy(
13215 &crate::support::compact_256::EXPAND_64[bits],
13216 );
13217 let reordered =
13218 _mm256_permutevar8x32_epi32(_mm256_castpd_si256(values.into()), control);
13219 let result = _mm256_and_si256(reordered, mask.into());
13220 _mm256_castsi256_pd(result).simd_into(token)
13221 }
13222 );
13223 kernel(self, values, mask)
13224 }
13225 #[inline(always)]
13226 fn expand_merge_f64x4(
13227 self,
13228 values: f64x4<Self>,
13229 mask: mask64x4<Self>,
13230 merge: f64x4<Self>,
13231 ) -> f64x4<Self> {
13232 crate::kernel!(
13233 #[inline(always)]
13234 fn kernel(
13235 token: Avx2,
13236 values: f64x4<Avx2>,
13237 mask: mask64x4<Avx2>,
13238 merge: f64x4<Avx2>,
13239 ) -> f64x4<Avx2> {
13240 let bits = token.to_bitmask_mask64x4(mask) as usize & 15;
13241 let control: __m256i = crate::transmute::checked_transmute_copy(
13242 &crate::support::compact_256::EXPAND_64[bits],
13243 );
13244 let reordered =
13245 _mm256_permutevar8x32_epi32(_mm256_castpd_si256(values.into()), control);
13246 let result =
13247 _mm256_blendv_epi8(_mm256_castpd_si256(merge.into()), reordered, mask.into());
13248 _mm256_castsi256_pd(result).simd_into(token)
13249 }
13250 );
13251 kernel(self, values, mask, merge)
13252 }
13253 #[inline(always)]
13254 fn neg_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13255 crate::kernel!(
13256 #[inline(always)]
13257 fn kernel(token: Avx2, a: f64x4<Avx2>) -> f64x4<Avx2> {
13258 _mm256_xor_pd(a.into(), _mm256_set1_pd(-0.0)).simd_into(token)
13259 }
13260 );
13261 kernel(self, a)
13262 }
13263 #[inline(always)]
13264 fn sqrt_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13265 crate::kernel!(
13266 #[inline(always)]
13267 fn kernel(token: Avx2, a: f64x4<Avx2>) -> f64x4<Avx2> {
13268 _mm256_sqrt_pd(a.into()).simd_into(token)
13269 }
13270 );
13271 kernel(self, a)
13272 }
13273 #[inline(always)]
13274 fn approximate_recip_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13275 1.0 / a
13276 }
13277 #[inline(always)]
13278 fn add_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13279 crate::kernel!(
13280 #[inline(always)]
13281 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
13282 _mm256_add_pd(a.into(), b.into()).simd_into(token)
13283 }
13284 );
13285 kernel(self, a, b)
13286 }
13287 #[inline(always)]
13288 fn sub_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13289 crate::kernel!(
13290 #[inline(always)]
13291 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
13292 _mm256_sub_pd(a.into(), b.into()).simd_into(token)
13293 }
13294 );
13295 kernel(self, a, b)
13296 }
13297 #[inline(always)]
13298 fn mul_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13299 crate::kernel!(
13300 #[inline(always)]
13301 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
13302 _mm256_mul_pd(a.into(), b.into()).simd_into(token)
13303 }
13304 );
13305 kernel(self, a, b)
13306 }
13307 #[inline(always)]
13308 fn div_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13309 crate::kernel!(
13310 #[inline(always)]
13311 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
13312 _mm256_div_pd(a.into(), b.into()).simd_into(token)
13313 }
13314 );
13315 kernel(self, a, b)
13316 }
13317 #[inline(always)]
13318 fn copysign_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13319 crate::kernel!(
13320 #[inline(always)]
13321 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
13322 let mask = _mm256_set1_pd(-0.0);
13323 _mm256_or_pd(
13324 _mm256_and_pd(mask, b.into()),
13325 _mm256_andnot_pd(mask, a.into()),
13326 )
13327 .simd_into(token)
13328 }
13329 );
13330 kernel(self, a, b)
13331 }
13332 #[inline(always)]
13333 fn max_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13334 crate::kernel!(
13335 #[inline(always)]
13336 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
13337 _mm256_max_pd(a.into(), b.into()).simd_into(token)
13338 }
13339 );
13340 kernel(self, a, b)
13341 }
13342 #[inline(always)]
13343 fn min_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13344 crate::kernel!(
13345 #[inline(always)]
13346 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
13347 _mm256_min_pd(a.into(), b.into()).simd_into(token)
13348 }
13349 );
13350 kernel(self, a, b)
13351 }
13352 #[inline(always)]
13353 fn max_precise_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13354 crate::kernel!(
13355 #[inline(always)]
13356 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
13357 let intermediate = _mm256_max_pd(a.into(), b.into());
13358 let b_is_nan = _mm256_cmp_pd::<3i32>(b.into(), b.into());
13359 _mm256_blendv_pd(intermediate, a.into(), b_is_nan).simd_into(token)
13360 }
13361 );
13362 kernel(self, a, b)
13363 }
13364 #[inline(always)]
13365 fn min_precise_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13366 crate::kernel!(
13367 #[inline(always)]
13368 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
13369 let intermediate = _mm256_min_pd(a.into(), b.into());
13370 let b_is_nan = _mm256_cmp_pd::<3i32>(b.into(), b.into());
13371 _mm256_blendv_pd(intermediate, a.into(), b_is_nan).simd_into(token)
13372 }
13373 );
13374 kernel(self, a, b)
13375 }
13376 #[inline(always)]
13377 fn clamp_f64x4(self, a: f64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
13378 crate::kernel!(
13379 #[inline(always)]
13380 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>, c: f64x4<Avx2>) -> f64x4<Avx2> {
13381 debug_assert!(
13382 b.simd_le(c).all_true(),
13383 "clamp requires min <= max for all elements"
13384 );
13385 _mm256_min_pd(c.into(), _mm256_max_pd(b.into(), a.into())).simd_into(token)
13386 }
13387 );
13388 kernel(self, a, b, c)
13389 }
13390 #[inline(always)]
13391 fn simd_eq_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
13392 crate::kernel!(
13393 #[inline(always)]
13394 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> mask64x4<Avx2> {
13395 _mm256_castpd_si256(_mm256_cmp_pd::<0i32>(a.into(), b.into())).simd_into(token)
13396 }
13397 );
13398 kernel(self, a, b)
13399 }
13400 #[inline(always)]
13401 fn simd_ne_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
13402 crate::kernel!(
13403 #[inline(always)]
13404 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> mask64x4<Avx2> {
13405 _mm256_castpd_si256(_mm256_cmp_pd::<4i32>(a.into(), b.into())).simd_into(token)
13406 }
13407 );
13408 kernel(self, a, b)
13409 }
13410 #[inline(always)]
13411 fn simd_lt_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
13412 crate::kernel!(
13413 #[inline(always)]
13414 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> mask64x4<Avx2> {
13415 _mm256_castpd_si256(_mm256_cmp_pd::<17i32>(a.into(), b.into())).simd_into(token)
13416 }
13417 );
13418 kernel(self, a, b)
13419 }
13420 #[inline(always)]
13421 fn simd_le_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
13422 crate::kernel!(
13423 #[inline(always)]
13424 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> mask64x4<Avx2> {
13425 _mm256_castpd_si256(_mm256_cmp_pd::<18i32>(a.into(), b.into())).simd_into(token)
13426 }
13427 );
13428 kernel(self, a, b)
13429 }
13430 #[inline(always)]
13431 fn zip_low_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13432 crate::kernel!(
13433 #[inline(always)]
13434 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
13435 let lo = _mm256_unpacklo_pd(a.into(), b.into());
13436 let hi = _mm256_unpackhi_pd(a.into(), b.into());
13437 _mm256_permute2f128_pd::<0b0010_0000>(lo, hi).simd_into(token)
13438 }
13439 );
13440 kernel(self, a, b)
13441 }
13442 #[inline(always)]
13443 fn zip_high_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13444 crate::kernel!(
13445 #[inline(always)]
13446 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
13447 let lo = _mm256_unpacklo_pd(a.into(), b.into());
13448 let hi = _mm256_unpackhi_pd(a.into(), b.into());
13449 _mm256_permute2f128_pd::<0b0011_0001>(lo, hi).simd_into(token)
13450 }
13451 );
13452 kernel(self, a, b)
13453 }
13454 #[inline(always)]
13455 fn unzip_low_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13456 crate::kernel!(
13457 #[inline(always)]
13458 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
13459 let t1 = _mm256_permute4x64_pd::<0b11_01_10_00>(a.into());
13460 let t2 = _mm256_permute4x64_pd::<0b11_01_10_00>(b.into());
13461 _mm256_permute2f128_pd::<0b0010_0000>(t1, t2).simd_into(token)
13462 }
13463 );
13464 kernel(self, a, b)
13465 }
13466 #[inline(always)]
13467 fn unzip_high_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
13468 crate::kernel!(
13469 #[inline(always)]
13470 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
13471 let t1 = _mm256_permute4x64_pd::<0b11_01_10_00>(a.into());
13472 let t2 = _mm256_permute4x64_pd::<0b11_01_10_00>(b.into());
13473 _mm256_permute2f128_pd::<0b0011_0001>(t1, t2).simd_into(token)
13474 }
13475 );
13476 kernel(self, a, b)
13477 }
13478 #[inline(always)]
13479 fn interleave_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> (f64x4<Self>, f64x4<Self>) {
13480 crate::kernel!(
13481 #[inline(always)]
13482 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> (f64x4<Avx2>, f64x4<Avx2>) {
13483 let lo = _mm256_unpacklo_pd(a.into(), b.into());
13484 let hi = _mm256_unpackhi_pd(a.into(), b.into());
13485 (
13486 _mm256_permute2f128_pd::<0b0010_0000>(lo, hi).simd_into(token),
13487 _mm256_permute2f128_pd::<0b0011_0001>(lo, hi).simd_into(token),
13488 )
13489 }
13490 );
13491 kernel(self, a, b)
13492 }
13493 #[inline(always)]
13494 fn deinterleave_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> (f64x4<Self>, f64x4<Self>) {
13495 crate::kernel!(
13496 #[inline(always)]
13497 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> (f64x4<Avx2>, f64x4<Avx2>) {
13498 let t1 = _mm256_permute4x64_pd::<0b11_01_10_00>(a.into());
13499 let t2 = _mm256_permute4x64_pd::<0b11_01_10_00>(b.into());
13500 (
13501 _mm256_permute2f128_pd::<0b0010_0000>(t1, t2).simd_into(token),
13502 _mm256_permute2f128_pd::<0b0011_0001>(t1, t2).simd_into(token),
13503 )
13504 }
13505 );
13506 kernel(self, a, b)
13507 }
13508 #[inline(always)]
13509 fn mul_add_f64x4(self, a: f64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
13510 crate::kernel!(
13511 #[inline(always)]
13512 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>, c: f64x4<Avx2>) -> f64x4<Avx2> {
13513 _mm256_fmadd_pd(a.into(), b.into(), c.into()).simd_into(token)
13514 }
13515 );
13516 kernel(self, a, b, c)
13517 }
13518 #[inline(always)]
13519 fn mul_add_precise_f64x4(self, a: f64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
13520 self.mul_add_f64x4(a, b, c)
13521 }
13522 #[inline(always)]
13523 fn mul_sub_f64x4(self, a: f64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
13524 crate::kernel!(
13525 #[inline(always)]
13526 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>, c: f64x4<Avx2>) -> f64x4<Avx2> {
13527 _mm256_fmsub_pd(a.into(), b.into(), c.into()).simd_into(token)
13528 }
13529 );
13530 kernel(self, a, b, c)
13531 }
13532 #[inline(always)]
13533 fn mul_sub_precise_f64x4(self, a: f64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
13534 self.mul_sub_f64x4(a, b, c)
13535 }
13536 #[inline(always)]
13537 fn floor_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13538 crate::kernel!(
13539 #[inline(always)]
13540 fn kernel(token: Avx2, a: f64x4<Avx2>) -> f64x4<Avx2> {
13541 _mm256_round_pd::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
13542 .simd_into(token)
13543 }
13544 );
13545 kernel(self, a)
13546 }
13547 #[inline(always)]
13548 fn ceil_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13549 crate::kernel!(
13550 #[inline(always)]
13551 fn kernel(token: Avx2, a: f64x4<Avx2>) -> f64x4<Avx2> {
13552 _mm256_round_pd::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
13553 .simd_into(token)
13554 }
13555 );
13556 kernel(self, a)
13557 }
13558 #[inline(always)]
13559 fn round_ties_even_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13560 crate::kernel!(
13561 #[inline(always)]
13562 fn kernel(token: Avx2, a: f64x4<Avx2>) -> f64x4<Avx2> {
13563 _mm256_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
13564 .simd_into(token)
13565 }
13566 );
13567 kernel(self, a)
13568 }
13569 #[inline(always)]
13570 fn round_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13571 let bias = self.splat_f64x4(const { f64::next_down(0.5) });
13572 (a + self.copysign_f64x4(bias, a)).trunc()
13573 }
13574 #[inline(always)]
13575 fn fract_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13576 a - self.trunc_f64x4(a)
13577 }
13578 #[inline(always)]
13579 fn trunc_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13580 crate::kernel!(
13581 #[inline(always)]
13582 fn kernel(token: Avx2, a: f64x4<Avx2>) -> f64x4<Avx2> {
13583 _mm256_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
13584 .simd_into(token)
13585 }
13586 );
13587 kernel(self, a)
13588 }
13589 #[inline(always)]
13590 fn to_degrees_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13591 a * const { f64::to_degrees(1.0) }
13592 }
13593 #[inline(always)]
13594 fn to_radians_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
13595 a * const { f64::to_radians(1.0) }
13596 }
13597 #[inline(always)]
13598 fn is_nan_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
13599 a.simd_ne(a)
13600 }
13601 #[inline(always)]
13602 fn is_infinite_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
13603 a.abs().simd_eq(f64::INFINITY)
13604 }
13605 #[inline(always)]
13606 fn is_finite_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
13607 a.abs().simd_lt(f64::INFINITY)
13608 }
13609 #[inline(always)]
13610 fn is_subnormal_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
13611 let i: i64x4<Self> = a.bitcast();
13612 (i & 0x7FF0_0000_0000_0000_i64).simd_eq(0) & (i & 0x000F_FFFF_FFFF_FFFF_i64).simd_ne(0)
13613 }
13614 #[inline(always)]
13615 fn is_normal_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
13616 let i: i64x4<Self> = a.bitcast();
13617 let exp = i & 0x7FF0_0000_0000_0000_i64;
13618 exp.simd_ne(0) & exp.simd_ne(0x7FF0_0000_0000_0000_i64)
13619 }
13620 #[inline(always)]
13621 fn is_sign_positive_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
13622 let i: i64x4<Self> = a.bitcast();
13623 (i & i64::MIN).simd_eq(0)
13624 }
13625 #[inline(always)]
13626 fn is_sign_negative_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
13627 !a.is_sign_positive()
13628 }
13629 #[inline(always)]
13630 fn select_f64x4(self, a: mask64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
13631 crate::kernel!(
13632 #[inline(always)]
13633 fn kernel(
13634 token: Avx2,
13635 a: mask64x4<Avx2>,
13636 b: f64x4<Avx2>,
13637 c: f64x4<Avx2>,
13638 ) -> f64x4<Avx2> {
13639 _mm256_blendv_pd(c.into(), b.into(), _mm256_castsi256_pd(a.into())).simd_into(token)
13640 }
13641 );
13642 kernel(self, a, b, c)
13643 }
13644 #[inline(always)]
13645 fn combine_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x8<Self> {
13646 f64x8 {
13647 val: crate::support::Aligned512([a.val.0, b.val.0]),
13648 simd: self,
13649 }
13650 }
13651 #[inline(always)]
13652 fn split_f64x4(self, a: f64x4<Self>) -> (f64x2<Self>, f64x2<Self>) {
13653 crate::kernel!(
13654 #[inline(always)]
13655 fn kernel(token: Avx2, a: f64x4<Avx2>) -> (f64x2<Avx2>, f64x2<Avx2>) {
13656 (
13657 _mm256_extractf128_pd::<0>(a.into()).simd_into(token),
13658 _mm256_extractf128_pd::<1>(a.into()).simd_into(token),
13659 )
13660 }
13661 );
13662 kernel(self, a)
13663 }
13664 #[inline(always)]
13665 fn narrow_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f32x8<Self> {
13666 crate::kernel!(
13667 #[inline(always)]
13668 fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f32x8<Avx2> {
13669 let low = _mm256_cvtpd_ps(a.into());
13670 let high = _mm256_cvtpd_ps(b.into());
13671 _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(low), high).simd_into(token)
13672 }
13673 );
13674 kernel(self, a, b)
13675 }
13676 #[inline(always)]
13677 fn saturating_narrow_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f32x8<Self> {
13678 self.narrow_f64x4(a, b)
13679 }
13680 #[inline(always)]
13681 fn relaxed_narrow_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f32x8<Self> {
13682 self.narrow_f64x4(a, b)
13683 }
13684 #[inline(always)]
13685 fn cvt_u64_f64x4(self, a: f64x4<Self>) -> u64x4<Self> {
13686 crate::kernel!(
13687 #[inline(always)]
13688 fn kernel(token: Avx2, a: f64x4<Avx2>) -> u64x4<Avx2> {
13689 let a = a.into();
13690 let bits = _mm256_castpd_si256(a);
13691 let exponent = _mm256_srli_epi64::<52>(bits);
13692 let significand = _mm256_or_si256(
13693 _mm256_and_si256(bits, _mm256_set1_epi64x(0x000f_ffff_ffff_ffff)),
13694 _mm256_set1_epi64x(0x0010_0000_0000_0000),
13695 );
13696 let shift_bias = _mm256_set1_epi64x(1075);
13697 let right_count = _mm256_sub_epi64(shift_bias, exponent);
13698 let left_count = _mm256_sub_epi64(exponent, shift_bias);
13699 let converted = _mm256_or_si256(
13700 _mm256_srlv_epi64(significand, right_count),
13701 _mm256_sllv_epi64(significand, left_count),
13702 );
13703 converted.simd_into(token)
13704 }
13705 );
13706 kernel(self, a)
13707 }
13708 #[inline(always)]
13709 fn cvt_u64_precise_f64x4(self, a: f64x4<Self>) -> u64x4<Self> {
13710 crate::kernel!(
13711 #[inline(always)]
13712 fn kernel(token: Avx2, a: f64x4<Avx2>) -> u64x4<Avx2> {
13713 let a = a.into();
13714 let bits = _mm256_castpd_si256(a);
13715 let exponent = _mm256_srli_epi64::<52>(bits);
13716 let significand = _mm256_or_si256(
13717 _mm256_and_si256(bits, _mm256_set1_epi64x(0x000f_ffff_ffff_ffff)),
13718 _mm256_set1_epi64x(0x0010_0000_0000_0000),
13719 );
13720 let shift_bias = _mm256_set1_epi64x(1075);
13721 let right_count = _mm256_sub_epi64(shift_bias, exponent);
13722 let left_count = _mm256_sub_epi64(exponent, shift_bias);
13723 let converted = _mm256_or_si256(
13724 _mm256_srlv_epi64(significand, right_count),
13725 _mm256_sllv_epi64(significand, left_count),
13726 );
13727 let overflow = _mm256_castpd_si256(_mm256_cmp_pd::<29i32>(
13728 a,
13729 _mm256_set1_pd(18_446_744_073_709_551_616.0),
13730 ));
13731 _mm256_or_si256(converted, overflow).simd_into(token)
13732 }
13733 );
13734 kernel(self, a)
13735 }
13736 #[inline(always)]
13737 fn cvt_i64_f64x4(self, a: f64x4<Self>) -> i64x4<Self> {
13738 crate::kernel!(
13739 #[inline(always)]
13740 fn kernel(token: Avx2, a: f64x4<Avx2>) -> i64x4<Avx2> {
13741 let a = a.into();
13742 let bits = _mm256_castpd_si256(a);
13743 let absolute = _mm256_and_si256(bits, _mm256_set1_epi64x(i64::MAX));
13744 let exponent = _mm256_srli_epi64::<52>(absolute);
13745 let significand = _mm256_or_si256(
13746 _mm256_and_si256(bits, _mm256_set1_epi64x(0x000f_ffff_ffff_ffff)),
13747 _mm256_set1_epi64x(0x0010_0000_0000_0000),
13748 );
13749 let shift_bias = _mm256_set1_epi64x(1075);
13750 let right_count = _mm256_sub_epi64(shift_bias, exponent);
13751 let left_count = _mm256_sub_epi64(exponent, shift_bias);
13752 let magnitude = _mm256_or_si256(
13753 _mm256_srlv_epi64(significand, right_count),
13754 _mm256_sllv_epi64(significand, left_count),
13755 );
13756 let sign = _mm256_cmpgt_epi64(_mm256_setzero_si256(), bits);
13757 let converted = _mm256_sub_epi64(_mm256_xor_si256(magnitude, sign), sign);
13758 converted.simd_into(token)
13759 }
13760 );
13761 kernel(self, a)
13762 }
13763 #[inline(always)]
13764 fn cvt_i64_precise_f64x4(self, a: f64x4<Self>) -> i64x4<Self> {
13765 crate::kernel!(
13766 #[inline(always)]
13767 fn kernel(token: Avx2, a: f64x4<Avx2>) -> i64x4<Avx2> {
13768 let a = a.into();
13769 let bits = _mm256_castpd_si256(a);
13770 let absolute = _mm256_and_si256(bits, _mm256_set1_epi64x(i64::MAX));
13771 let exponent = _mm256_srli_epi64::<52>(absolute);
13772 let significand = _mm256_or_si256(
13773 _mm256_and_si256(bits, _mm256_set1_epi64x(0x000f_ffff_ffff_ffff)),
13774 _mm256_set1_epi64x(0x0010_0000_0000_0000),
13775 );
13776 let shift_bias = _mm256_set1_epi64x(1075);
13777 let right_count = _mm256_sub_epi64(shift_bias, exponent);
13778 let left_count = _mm256_sub_epi64(exponent, shift_bias);
13779 let magnitude = _mm256_or_si256(
13780 _mm256_srlv_epi64(significand, right_count),
13781 _mm256_sllv_epi64(significand, left_count),
13782 );
13783 let sign = _mm256_cmpgt_epi64(_mm256_setzero_si256(), bits);
13784 let converted = _mm256_sub_epi64(_mm256_xor_si256(magnitude, sign), sign);
13785 let overflow = _mm256_castpd_si256(_mm256_cmp_pd::<29i32>(
13786 _mm256_castsi256_pd(absolute),
13787 _mm256_set1_pd(9_223_372_036_854_775_808.0),
13788 ));
13789 let bound = _mm256_xor_si256(_mm256_set1_epi64x(i64::MAX), sign);
13790 _mm256_blendv_epi8(converted, bound, overflow).simd_into(token)
13791 }
13792 );
13793 kernel(self, a)
13794 }
13795 #[inline(always)]
13796 fn abs_i64x4(self, a: i64x4<Self>) -> i64x4<Self> {
13797 crate::kernel!(
13798 #[inline(always)]
13799 fn kernel(token: Avx2, a: i64x4<Avx2>) -> i64x4<Avx2> {
13800 let a = a.into();
13801 let mask = _mm256_cmpgt_epi64(_mm256_setzero_si256(), a);
13802 _mm256_sub_epi64(_mm256_xor_si256(a, mask), mask).simd_into(token)
13803 }
13804 );
13805 kernel(self, a)
13806 }
13807 #[inline(always)]
13808 fn splat_i64x4(self, val: i64) -> i64x4<Self> {
13809 crate::kernel!(
13810 #[inline(always)]
13811 fn kernel(token: Avx2, val: i64) -> i64x4<Avx2> {
13812 _mm256_set1_epi64x(val).simd_into(token)
13813 }
13814 );
13815 kernel(self, val)
13816 }
13817 #[inline(always)]
13818 fn reverse_i64x4(self, a: i64x4<Self>) -> i64x4<Self> {
13819 let indices: u8x32<Self> = [
13820 24, 25, 26, 27, 28, 29, 30, 31, 16, 17, 18, 19, 20, 21, 22, 23, 8, 9, 10, 11, 12, 13,
13821 14, 15, 0, 1, 2, 3, 4, 5, 6, 7,
13822 ]
13823 .simd_into(self);
13824 self.swizzle_dyn_i64x4(a, indices)
13825 }
13826 #[inline(always)]
13827 fn slide_i64x4<const SHIFT: usize>(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
13828 if SHIFT >= 4usize {
13829 return b;
13830 }
13831 let result = cross_block_alignr_256x1(
13832 self,
13833 Bytes::to_bytes(b).val.0,
13834 Bytes::to_bytes(a).val.0,
13835 SHIFT * 8usize,
13836 );
13837 Bytes::from_bytes(u8x32 {
13838 val: crate::support::Aligned256(result),
13839 simd: self,
13840 })
13841 }
13842 #[inline(always)]
13843 fn slide_within_blocks_i64x4<const SHIFT: usize>(
13844 self,
13845 a: i64x4<Self>,
13846 b: i64x4<Self>,
13847 ) -> i64x4<Self> {
13848 if SHIFT >= 2usize {
13849 return b;
13850 }
13851 let result = dyn_alignr_256(
13852 self,
13853 Bytes::to_bytes(b).val.0,
13854 Bytes::to_bytes(a).val.0,
13855 SHIFT * 8usize,
13856 );
13857 Bytes::from_bytes(u8x32 {
13858 val: crate::support::Aligned256(result),
13859 simd: self,
13860 })
13861 }
13862 #[inline(always)]
13863 fn compress_i64x4(self, values: i64x4<Self>, mask: mask64x4<Self>) -> i64x4<Self> {
13864 crate::kernel!(
13865 #[inline(always)]
13866 fn kernel(token: Avx2, values: i64x4<Avx2>, mask: mask64x4<Avx2>) -> i64x4<Avx2> {
13867 let bits = token.to_bitmask_mask64x4(mask) as usize & 15;
13868 let control: __m256i = crate::transmute::checked_transmute_copy(
13869 &crate::support::compact_256::COMPRESS_64[bits],
13870 );
13871 let reordered = _mm256_permutevar8x32_epi32(values.into(), control);
13872 let result = _mm256_andnot_si256(_mm256_srai_epi32::<31>(control), reordered);
13873 result.simd_into(token)
13874 }
13875 );
13876 kernel(self, values, mask)
13877 }
13878 #[inline(always)]
13879 fn compress_merge_i64x4(
13880 self,
13881 values: i64x4<Self>,
13882 mask: mask64x4<Self>,
13883 merge: i64x4<Self>,
13884 ) -> i64x4<Self> {
13885 crate::kernel!(
13886 #[inline(always)]
13887 fn kernel(
13888 token: Avx2,
13889 values: i64x4<Avx2>,
13890 mask: mask64x4<Avx2>,
13891 merge: i64x4<Avx2>,
13892 ) -> i64x4<Avx2> {
13893 let bits = token.to_bitmask_mask64x4(mask) as usize & 15;
13894 let control: __m256i = crate::transmute::checked_transmute_copy(
13895 &crate::support::compact_256::COMPRESS_64[bits],
13896 );
13897 let reordered = _mm256_permutevar8x32_epi32(values.into(), control);
13898 let result = _mm256_blendv_epi8(reordered, merge.into(), control);
13899 result.simd_into(token)
13900 }
13901 );
13902 kernel(self, values, mask, merge)
13903 }
13904 #[inline(always)]
13905 fn expand_i64x4(self, values: i64x4<Self>, mask: mask64x4<Self>) -> i64x4<Self> {
13906 crate::kernel!(
13907 #[inline(always)]
13908 fn kernel(token: Avx2, values: i64x4<Avx2>, mask: mask64x4<Avx2>) -> i64x4<Avx2> {
13909 let bits = token.to_bitmask_mask64x4(mask) as usize & 15;
13910 let control: __m256i = crate::transmute::checked_transmute_copy(
13911 &crate::support::compact_256::EXPAND_64[bits],
13912 );
13913 let reordered = _mm256_permutevar8x32_epi32(values.into(), control);
13914 let result = _mm256_and_si256(reordered, mask.into());
13915 result.simd_into(token)
13916 }
13917 );
13918 kernel(self, values, mask)
13919 }
13920 #[inline(always)]
13921 fn expand_merge_i64x4(
13922 self,
13923 values: i64x4<Self>,
13924 mask: mask64x4<Self>,
13925 merge: i64x4<Self>,
13926 ) -> i64x4<Self> {
13927 crate::kernel!(
13928 #[inline(always)]
13929 fn kernel(
13930 token: Avx2,
13931 values: i64x4<Avx2>,
13932 mask: mask64x4<Avx2>,
13933 merge: i64x4<Avx2>,
13934 ) -> i64x4<Avx2> {
13935 let bits = token.to_bitmask_mask64x4(mask) as usize & 15;
13936 let control: __m256i = crate::transmute::checked_transmute_copy(
13937 &crate::support::compact_256::EXPAND_64[bits],
13938 );
13939 let reordered = _mm256_permutevar8x32_epi32(values.into(), control);
13940 let result = _mm256_blendv_epi8(merge.into(), reordered, mask.into());
13941 result.simd_into(token)
13942 }
13943 );
13944 kernel(self, values, mask, merge)
13945 }
13946 #[inline(always)]
13947 fn count_ones_i64x4(self, a: i64x4<Self>) -> i64x4<Self> {
13948 crate::kernel!(
13949 #[inline(always)]
13950 fn kernel(token: Avx2, a: i64x4<Avx2>) -> i64x4<Avx2> {
13951 let value = a.into();
13952 let nibble_mask = _mm256_set1_epi8(0x0f);
13953 let lookup = _mm256_setr_epi8(
13954 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 0, 1, 1, 2, 1, 2, 2, 3, 1, 2,
13955 2, 3, 2, 3, 3, 4,
13956 );
13957 let low = _mm256_and_si256(value, nibble_mask);
13958 let high = _mm256_and_si256(_mm256_srli_epi16::<4>(value), nibble_mask);
13959 let byte_counts = _mm256_add_epi8(
13960 _mm256_shuffle_epi8(lookup, low),
13961 _mm256_shuffle_epi8(lookup, high),
13962 );
13963 _mm256_sad_epu8(byte_counts, _mm256_setzero_si256()).simd_into(token)
13964 }
13965 );
13966 kernel(self, a)
13967 }
13968 #[inline(always)]
13969 fn count_zeros_i64x4(self, a: i64x4<Self>) -> i64x4<Self> {
13970 self.count_ones_i64x4(self.not_i64x4(a))
13971 }
13972 #[inline(always)]
13973 fn add_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
13974 crate::kernel!(
13975 #[inline(always)]
13976 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
13977 _mm256_add_epi64(a.into(), b.into()).simd_into(token)
13978 }
13979 );
13980 kernel(self, a, b)
13981 }
13982 #[inline(always)]
13983 fn saturating_add_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
13984 crate::kernel!(
13985 #[inline(always)]
13986 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
13987 let a = a.into();
13988 let b = b.into();
13989 let wrapped = _mm256_add_epi64(a, b);
13990 let overflow = _mm256_xor_si256(_mm256_cmpgt_epi64(a, wrapped), b);
13991 let bound =
13992 _mm256_add_epi64(_mm256_srli_epi64::<63>(a), _mm256_set1_epi64x(i64::MAX));
13993 let result = _mm256_blendv_pd(
13994 _mm256_castsi256_pd(wrapped),
13995 _mm256_castsi256_pd(bound),
13996 _mm256_castsi256_pd(overflow),
13997 );
13998 _mm256_castpd_si256(result).simd_into(token)
13999 }
14000 );
14001 kernel(self, a, b)
14002 }
14003 #[inline(always)]
14004 fn sub_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14005 crate::kernel!(
14006 #[inline(always)]
14007 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
14008 _mm256_sub_epi64(a.into(), b.into()).simd_into(token)
14009 }
14010 );
14011 kernel(self, a, b)
14012 }
14013 #[inline(always)]
14014 fn saturating_sub_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14015 crate::kernel!(
14016 #[inline(always)]
14017 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
14018 let a = a.into();
14019 let b = b.into();
14020 let wrapped = _mm256_sub_epi64(a, b);
14021 let overflow = _mm256_xor_si256(_mm256_cmpgt_epi64(wrapped, a), b);
14022 let bound =
14023 _mm256_add_epi64(_mm256_srli_epi64::<63>(a), _mm256_set1_epi64x(i64::MAX));
14024 let result = _mm256_blendv_pd(
14025 _mm256_castsi256_pd(wrapped),
14026 _mm256_castsi256_pd(bound),
14027 _mm256_castsi256_pd(overflow),
14028 );
14029 _mm256_castpd_si256(result).simd_into(token)
14030 }
14031 );
14032 kernel(self, a, b)
14033 }
14034 #[inline(always)]
14035 fn mul_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14036 crate::kernel!(
14037 #[inline(always)]
14038 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
14039 let a = a.into();
14040 let b = b.into();
14041 let a_high = _mm256_shuffle_epi32::<0xf5>(a);
14042 let b_high = _mm256_shuffle_epi32::<0xf5>(b);
14043 let cross =
14044 _mm256_add_epi64(_mm256_mul_epu32(a_high, b), _mm256_mul_epu32(a, b_high));
14045 let low = _mm256_mul_epu32(a, b);
14046 _mm256_add_epi64(low, _mm256_slli_epi64::<32>(cross)).simd_into(token)
14047 }
14048 );
14049 kernel(self, a, b)
14050 }
14051 #[inline(always)]
14052 fn and_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14053 crate::kernel!(
14054 #[inline(always)]
14055 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
14056 _mm256_and_si256(a.into(), b.into()).simd_into(token)
14057 }
14058 );
14059 kernel(self, a, b)
14060 }
14061 #[inline(always)]
14062 fn or_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14063 crate::kernel!(
14064 #[inline(always)]
14065 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
14066 _mm256_or_si256(a.into(), b.into()).simd_into(token)
14067 }
14068 );
14069 kernel(self, a, b)
14070 }
14071 #[inline(always)]
14072 fn xor_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14073 crate::kernel!(
14074 #[inline(always)]
14075 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
14076 _mm256_xor_si256(a.into(), b.into()).simd_into(token)
14077 }
14078 );
14079 kernel(self, a, b)
14080 }
14081 #[inline(always)]
14082 fn not_i64x4(self, a: i64x4<Self>) -> i64x4<Self> {
14083 a ^ !0
14084 }
14085 #[inline(always)]
14086 fn shl_i64x4(self, a: i64x4<Self>, shift: u32) -> i64x4<Self> {
14087 crate::kernel!(
14088 #[inline(always)]
14089 fn kernel(token: Avx2, a: i64x4<Avx2>, shift: u32) -> i64x4<Avx2> {
14090 _mm256_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
14091 }
14092 );
14093 kernel(self, a, shift)
14094 }
14095 #[inline(always)]
14096 fn shlv_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14097 crate::kernel!(
14098 #[inline(always)]
14099 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
14100 _mm256_sllv_epi64(a.into(), b.into()).simd_into(token)
14101 }
14102 );
14103 kernel(self, a, b)
14104 }
14105 #[inline(always)]
14106 fn shr_i64x4(self, a: i64x4<Self>, shift: u32) -> i64x4<Self> {
14107 crate::kernel!(
14108 #[inline(always)]
14109 fn kernel(token: Avx2, a: i64x4<Avx2>, shift: u32) -> i64x4<Avx2> {
14110 let value = a.into();
14111 let count = _mm_cvtsi32_si128(shift.cast_signed());
14112 let shifted_bias = _mm256_srl_epi64(_mm256_set1_epi64x(i64::MIN), count);
14113 let shifted = _mm256_srl_epi64(value, count);
14114 _mm256_sub_epi64(_mm256_xor_si256(shifted, shifted_bias), shifted_bias)
14115 .simd_into(token)
14116 }
14117 );
14118 kernel(self, a, shift)
14119 }
14120 #[inline(always)]
14121 fn shrv_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14122 crate::kernel!(
14123 #[inline(always)]
14124 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
14125 let value = a.into();
14126 let counts = b.into();
14127 let bias = _mm256_set1_epi64x(i64::MIN);
14128 let shifted_bias = _mm256_srlv_epi64(bias, counts);
14129 let shifted = _mm256_srlv_epi64(value, counts);
14130 _mm256_sub_epi64(_mm256_xor_si256(shifted, shifted_bias), shifted_bias)
14131 .simd_into(token)
14132 }
14133 );
14134 kernel(self, a, b)
14135 }
14136 #[inline(always)]
14137 fn max_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14138 crate::kernel!(
14139 #[inline(always)]
14140 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
14141 let a = a.into();
14142 let b = b.into();
14143 let mask = _mm256_cmpgt_epi64(b, a);
14144 let result = _mm256_blendv_pd(
14145 _mm256_castsi256_pd(a),
14146 _mm256_castsi256_pd(b),
14147 _mm256_castsi256_pd(mask),
14148 );
14149 _mm256_castpd_si256(result).simd_into(token)
14150 }
14151 );
14152 kernel(self, a, b)
14153 }
14154 #[inline(always)]
14155 fn min_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14156 crate::kernel!(
14157 #[inline(always)]
14158 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
14159 let a = a.into();
14160 let b = b.into();
14161 let mask = _mm256_cmpgt_epi64(a, b);
14162 let result = _mm256_blendv_pd(
14163 _mm256_castsi256_pd(a),
14164 _mm256_castsi256_pd(b),
14165 _mm256_castsi256_pd(mask),
14166 );
14167 _mm256_castpd_si256(result).simd_into(token)
14168 }
14169 );
14170 kernel(self, a, b)
14171 }
14172 #[inline(always)]
14173 fn clamp_i64x4(self, a: i64x4<Self>, b: i64x4<Self>, c: i64x4<Self>) -> i64x4<Self> {
14174 debug_assert!(
14175 b.simd_le(c).all_true(),
14176 "clamp requires min <= max for all elements"
14177 );
14178 a.max(b).min(c)
14179 }
14180 #[inline(always)]
14181 fn simd_eq_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> mask64x4<Self> {
14182 crate::kernel!(
14183 #[inline(always)]
14184 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> mask64x4<Avx2> {
14185 _mm256_cmpeq_epi64(a.into(), b.into()).simd_into(token)
14186 }
14187 );
14188 kernel(self, a, b)
14189 }
14190 #[inline(always)]
14191 fn simd_lt_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> mask64x4<Self> {
14192 crate::kernel!(
14193 #[inline(always)]
14194 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> mask64x4<Avx2> {
14195 _mm256_cmpgt_epi64(b.into(), a.into()).simd_into(token)
14196 }
14197 );
14198 kernel(self, a, b)
14199 }
14200 #[inline(always)]
14201 fn simd_le_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> mask64x4<Self> {
14202 crate::kernel!(
14203 #[inline(always)]
14204 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> mask64x4<Avx2> {
14205 _mm256_xor_si256(
14206 _mm256_cmpgt_epi64(a.into(), b.into()),
14207 _mm256_set1_epi64x(-1),
14208 )
14209 .simd_into(token)
14210 }
14211 );
14212 kernel(self, a, b)
14213 }
14214 #[inline(always)]
14215 fn zip_low_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14216 crate::kernel!(
14217 #[inline(always)]
14218 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
14219 let lo = _mm256_unpacklo_epi64(a.into(), b.into());
14220 let hi = _mm256_unpackhi_epi64(a.into(), b.into());
14221 _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token)
14222 }
14223 );
14224 kernel(self, a, b)
14225 }
14226 #[inline(always)]
14227 fn zip_high_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14228 crate::kernel!(
14229 #[inline(always)]
14230 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
14231 let lo = _mm256_unpacklo_epi64(a.into(), b.into());
14232 let hi = _mm256_unpackhi_epi64(a.into(), b.into());
14233 _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token)
14234 }
14235 );
14236 kernel(self, a, b)
14237 }
14238 #[inline(always)]
14239 fn unzip_low_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14240 crate::kernel!(
14241 #[inline(always)]
14242 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
14243 let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(a.into());
14244 let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(b.into());
14245 _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token)
14246 }
14247 );
14248 kernel(self, a, b)
14249 }
14250 #[inline(always)]
14251 fn unzip_high_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
14252 crate::kernel!(
14253 #[inline(always)]
14254 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
14255 let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(a.into());
14256 let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(b.into());
14257 _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token)
14258 }
14259 );
14260 kernel(self, a, b)
14261 }
14262 #[inline(always)]
14263 fn interleave_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> (i64x4<Self>, i64x4<Self>) {
14264 crate::kernel!(
14265 #[inline(always)]
14266 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> (i64x4<Avx2>, i64x4<Avx2>) {
14267 let lo = _mm256_unpacklo_epi64(a.into(), b.into());
14268 let hi = _mm256_unpackhi_epi64(a.into(), b.into());
14269 (
14270 _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token),
14271 _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token),
14272 )
14273 }
14274 );
14275 kernel(self, a, b)
14276 }
14277 #[inline(always)]
14278 fn deinterleave_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> (i64x4<Self>, i64x4<Self>) {
14279 crate::kernel!(
14280 #[inline(always)]
14281 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> (i64x4<Avx2>, i64x4<Avx2>) {
14282 let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(a.into());
14283 let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(b.into());
14284 (
14285 _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token),
14286 _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token),
14287 )
14288 }
14289 );
14290 kernel(self, a, b)
14291 }
14292 #[inline(always)]
14293 fn select_i64x4(self, a: mask64x4<Self>, b: i64x4<Self>, c: i64x4<Self>) -> i64x4<Self> {
14294 crate::kernel!(
14295 #[inline(always)]
14296 fn kernel(
14297 token: Avx2,
14298 a: mask64x4<Avx2>,
14299 b: i64x4<Avx2>,
14300 c: i64x4<Avx2>,
14301 ) -> i64x4<Avx2> {
14302 _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
14303 }
14304 );
14305 kernel(self, a, b, c)
14306 }
14307 #[inline(always)]
14308 fn combine_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x8<Self> {
14309 i64x8 {
14310 val: crate::support::Aligned512([a.val.0, b.val.0]),
14311 simd: self,
14312 }
14313 }
14314 #[inline(always)]
14315 fn split_i64x4(self, a: i64x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
14316 crate::kernel!(
14317 #[inline(always)]
14318 fn kernel(token: Avx2, a: i64x4<Avx2>) -> (i64x2<Avx2>, i64x2<Avx2>) {
14319 (
14320 _mm256_extracti128_si256::<0>(a.into()).simd_into(token),
14321 _mm256_extracti128_si256::<1>(a.into()).simd_into(token),
14322 )
14323 }
14324 );
14325 kernel(self, a)
14326 }
14327 #[inline(always)]
14328 fn neg_i64x4(self, a: i64x4<Self>) -> i64x4<Self> {
14329 crate::kernel!(
14330 #[inline(always)]
14331 fn kernel(token: Avx2, a: i64x4<Avx2>) -> i64x4<Avx2> {
14332 _mm256_sub_epi64(_mm256_setzero_si256(), a.into()).simd_into(token)
14333 }
14334 );
14335 kernel(self, a)
14336 }
14337 #[inline(always)]
14338 fn narrow_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i32x8<Self> {
14339 crate::kernel!(
14340 #[inline(always)]
14341 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i32x8<Avx2> {
14342 _mm256_permute4x64_epi64::<0xd8>(_mm256_castps_si256(_mm256_shuffle_ps::<0x88>(
14343 _mm256_castsi256_ps(a.into()),
14344 _mm256_castsi256_ps(b.into()),
14345 )))
14346 .simd_into(token)
14347 }
14348 );
14349 kernel(self, a, b)
14350 }
14351 #[inline(always)]
14352 fn saturating_narrow_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i32x8<Self> {
14353 crate::kernel!(
14354 #[inline(always)]
14355 fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i32x8<Avx2> {
14356 let a = a.into();
14357 let b = b.into();
14358 let low = _mm256_permute4x64_epi64::<0xd8>(_mm256_castps_si256(
14359 _mm256_shuffle_ps::<0x88>(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b)),
14360 ));
14361 let high = _mm256_permute4x64_epi64::<0xd8>(_mm256_castps_si256(
14362 _mm256_shuffle_ps::<0xdd>(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b)),
14363 ));
14364 let low_sign = _mm256_srai_epi32::<31>(low);
14365 let fits = _mm256_cmpeq_epi32(high, low_sign);
14366 let high_sign = _mm256_srai_epi32::<31>(high);
14367 let bound = _mm256_xor_si256(high_sign, _mm256_set1_epi32(i32::MAX));
14368 _mm256_blendv_epi8(bound, low, fits).simd_into(token)
14369 }
14370 );
14371 kernel(self, a, b)
14372 }
14373 #[inline(always)]
14374 fn relaxed_narrow_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i32x8<Self> {
14375 debug_assert!(
14376 a.as_slice()
14377 .iter()
14378 .chain(b.as_slice())
14379 .all(|&value| { value >= i32::MIN as i64 && value <= i32::MAX as i64 }),
14380 "relaxed_narrow inputs must fit in the destination type",
14381 );
14382 self.narrow_i64x4(a, b)
14383 }
14384 #[inline(always)]
14385 fn cvt_f64_i64x4(self, a: i64x4<Self>) -> f64x4<Self> {
14386 crate::kernel!(
14387 #[inline(always)]
14388 fn kernel(token: Avx2, a: i64x4<Avx2>) -> f64x4<Avx2> {
14389 let a = a.into();
14390 let low = _mm256_blend_epi32::<0xaa>(a, _mm256_set1_epi64x(0x4330_0000_0000_0000));
14391 let high = _mm256_srli_epi64::<32>(a);
14392 let high = _mm256_xor_si256(high, _mm256_set1_epi64x(4985484789646622720i64));
14393 let high = _mm256_sub_pd(
14394 _mm256_castsi256_pd(high),
14395 _mm256_set1_pd(f64::from_bits(4985484789647671296u64)),
14396 );
14397 _mm256_add_pd(_mm256_castsi256_pd(low), high).simd_into(token)
14398 }
14399 );
14400 kernel(self, a)
14401 }
14402 #[inline(always)]
14403 fn splat_u64x4(self, val: u64) -> u64x4<Self> {
14404 crate::kernel!(
14405 #[inline(always)]
14406 fn kernel(token: Avx2, val: u64) -> u64x4<Avx2> {
14407 _mm256_set1_epi64x(val.cast_signed()).simd_into(token)
14408 }
14409 );
14410 kernel(self, val)
14411 }
14412 #[inline(always)]
14413 fn reverse_u64x4(self, a: u64x4<Self>) -> u64x4<Self> {
14414 let indices: u8x32<Self> = [
14415 24, 25, 26, 27, 28, 29, 30, 31, 16, 17, 18, 19, 20, 21, 22, 23, 8, 9, 10, 11, 12, 13,
14416 14, 15, 0, 1, 2, 3, 4, 5, 6, 7,
14417 ]
14418 .simd_into(self);
14419 self.swizzle_dyn_u64x4(a, indices)
14420 }
14421 #[inline(always)]
14422 fn slide_u64x4<const SHIFT: usize>(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14423 if SHIFT >= 4usize {
14424 return b;
14425 }
14426 let result = cross_block_alignr_256x1(
14427 self,
14428 Bytes::to_bytes(b).val.0,
14429 Bytes::to_bytes(a).val.0,
14430 SHIFT * 8usize,
14431 );
14432 Bytes::from_bytes(u8x32 {
14433 val: crate::support::Aligned256(result),
14434 simd: self,
14435 })
14436 }
14437 #[inline(always)]
14438 fn slide_within_blocks_u64x4<const SHIFT: usize>(
14439 self,
14440 a: u64x4<Self>,
14441 b: u64x4<Self>,
14442 ) -> u64x4<Self> {
14443 if SHIFT >= 2usize {
14444 return b;
14445 }
14446 let result = dyn_alignr_256(
14447 self,
14448 Bytes::to_bytes(b).val.0,
14449 Bytes::to_bytes(a).val.0,
14450 SHIFT * 8usize,
14451 );
14452 Bytes::from_bytes(u8x32 {
14453 val: crate::support::Aligned256(result),
14454 simd: self,
14455 })
14456 }
14457 #[inline(always)]
14458 fn compress_u64x4(self, values: u64x4<Self>, mask: mask64x4<Self>) -> u64x4<Self> {
14459 crate::kernel!(
14460 #[inline(always)]
14461 fn kernel(token: Avx2, values: u64x4<Avx2>, mask: mask64x4<Avx2>) -> u64x4<Avx2> {
14462 let bits = token.to_bitmask_mask64x4(mask) as usize & 15;
14463 let control: __m256i = crate::transmute::checked_transmute_copy(
14464 &crate::support::compact_256::COMPRESS_64[bits],
14465 );
14466 let reordered = _mm256_permutevar8x32_epi32(values.into(), control);
14467 let result = _mm256_andnot_si256(_mm256_srai_epi32::<31>(control), reordered);
14468 result.simd_into(token)
14469 }
14470 );
14471 kernel(self, values, mask)
14472 }
14473 #[inline(always)]
14474 fn compress_merge_u64x4(
14475 self,
14476 values: u64x4<Self>,
14477 mask: mask64x4<Self>,
14478 merge: u64x4<Self>,
14479 ) -> u64x4<Self> {
14480 crate::kernel!(
14481 #[inline(always)]
14482 fn kernel(
14483 token: Avx2,
14484 values: u64x4<Avx2>,
14485 mask: mask64x4<Avx2>,
14486 merge: u64x4<Avx2>,
14487 ) -> u64x4<Avx2> {
14488 let bits = token.to_bitmask_mask64x4(mask) as usize & 15;
14489 let control: __m256i = crate::transmute::checked_transmute_copy(
14490 &crate::support::compact_256::COMPRESS_64[bits],
14491 );
14492 let reordered = _mm256_permutevar8x32_epi32(values.into(), control);
14493 let result = _mm256_blendv_epi8(reordered, merge.into(), control);
14494 result.simd_into(token)
14495 }
14496 );
14497 kernel(self, values, mask, merge)
14498 }
14499 #[inline(always)]
14500 fn expand_u64x4(self, values: u64x4<Self>, mask: mask64x4<Self>) -> u64x4<Self> {
14501 crate::kernel!(
14502 #[inline(always)]
14503 fn kernel(token: Avx2, values: u64x4<Avx2>, mask: mask64x4<Avx2>) -> u64x4<Avx2> {
14504 let bits = token.to_bitmask_mask64x4(mask) as usize & 15;
14505 let control: __m256i = crate::transmute::checked_transmute_copy(
14506 &crate::support::compact_256::EXPAND_64[bits],
14507 );
14508 let reordered = _mm256_permutevar8x32_epi32(values.into(), control);
14509 let result = _mm256_and_si256(reordered, mask.into());
14510 result.simd_into(token)
14511 }
14512 );
14513 kernel(self, values, mask)
14514 }
14515 #[inline(always)]
14516 fn expand_merge_u64x4(
14517 self,
14518 values: u64x4<Self>,
14519 mask: mask64x4<Self>,
14520 merge: u64x4<Self>,
14521 ) -> u64x4<Self> {
14522 crate::kernel!(
14523 #[inline(always)]
14524 fn kernel(
14525 token: Avx2,
14526 values: u64x4<Avx2>,
14527 mask: mask64x4<Avx2>,
14528 merge: u64x4<Avx2>,
14529 ) -> u64x4<Avx2> {
14530 let bits = token.to_bitmask_mask64x4(mask) as usize & 15;
14531 let control: __m256i = crate::transmute::checked_transmute_copy(
14532 &crate::support::compact_256::EXPAND_64[bits],
14533 );
14534 let reordered = _mm256_permutevar8x32_epi32(values.into(), control);
14535 let result = _mm256_blendv_epi8(merge.into(), reordered, mask.into());
14536 result.simd_into(token)
14537 }
14538 );
14539 kernel(self, values, mask, merge)
14540 }
14541 #[inline(always)]
14542 fn count_ones_u64x4(self, a: u64x4<Self>) -> u64x4<Self> {
14543 crate::kernel!(
14544 #[inline(always)]
14545 fn kernel(token: Avx2, a: u64x4<Avx2>) -> u64x4<Avx2> {
14546 let value = a.into();
14547 let nibble_mask = _mm256_set1_epi8(0x0f);
14548 let lookup = _mm256_setr_epi8(
14549 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 0, 1, 1, 2, 1, 2, 2, 3, 1, 2,
14550 2, 3, 2, 3, 3, 4,
14551 );
14552 let low = _mm256_and_si256(value, nibble_mask);
14553 let high = _mm256_and_si256(_mm256_srli_epi16::<4>(value), nibble_mask);
14554 let byte_counts = _mm256_add_epi8(
14555 _mm256_shuffle_epi8(lookup, low),
14556 _mm256_shuffle_epi8(lookup, high),
14557 );
14558 _mm256_sad_epu8(byte_counts, _mm256_setzero_si256()).simd_into(token)
14559 }
14560 );
14561 kernel(self, a)
14562 }
14563 #[inline(always)]
14564 fn count_zeros_u64x4(self, a: u64x4<Self>) -> u64x4<Self> {
14565 self.count_ones_u64x4(self.not_u64x4(a))
14566 }
14567 #[inline(always)]
14568 fn add_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14569 crate::kernel!(
14570 #[inline(always)]
14571 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
14572 _mm256_add_epi64(a.into(), b.into()).simd_into(token)
14573 }
14574 );
14575 kernel(self, a, b)
14576 }
14577 #[inline(always)]
14578 fn saturating_add_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14579 crate::kernel!(
14580 #[inline(always)]
14581 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
14582 let a = a.into();
14583 let b = b.into();
14584 let wrapped = _mm256_add_epi64(a, b);
14585 let sign_bias = _mm256_set1_epi64x(i64::MIN);
14586 let overflow = _mm256_cmpgt_epi64(
14587 _mm256_xor_si256(a, sign_bias),
14588 _mm256_xor_si256(wrapped, sign_bias),
14589 );
14590 _mm256_or_si256(wrapped, overflow).simd_into(token)
14591 }
14592 );
14593 kernel(self, a, b)
14594 }
14595 #[inline(always)]
14596 fn sub_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14597 crate::kernel!(
14598 #[inline(always)]
14599 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
14600 _mm256_sub_epi64(a.into(), b.into()).simd_into(token)
14601 }
14602 );
14603 kernel(self, a, b)
14604 }
14605 #[inline(always)]
14606 fn saturating_sub_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14607 crate::kernel!(
14608 #[inline(always)]
14609 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
14610 let a = a.into();
14611 let b = b.into();
14612 let wrapped = _mm256_sub_epi64(a, b);
14613 let sign_bias = _mm256_set1_epi64x(i64::MIN);
14614 let no_borrow = _mm256_cmpgt_epi64(
14615 _mm256_xor_si256(a, sign_bias),
14616 _mm256_xor_si256(b, sign_bias),
14617 );
14618 _mm256_and_si256(wrapped, no_borrow).simd_into(token)
14619 }
14620 );
14621 kernel(self, a, b)
14622 }
14623 #[inline(always)]
14624 fn mul_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14625 crate::kernel!(
14626 #[inline(always)]
14627 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
14628 let a = a.into();
14629 let b = b.into();
14630 let a_high = _mm256_shuffle_epi32::<0xf5>(a);
14631 let b_high = _mm256_shuffle_epi32::<0xf5>(b);
14632 let cross =
14633 _mm256_add_epi64(_mm256_mul_epu32(a_high, b), _mm256_mul_epu32(a, b_high));
14634 let low = _mm256_mul_epu32(a, b);
14635 _mm256_add_epi64(low, _mm256_slli_epi64::<32>(cross)).simd_into(token)
14636 }
14637 );
14638 kernel(self, a, b)
14639 }
14640 #[inline(always)]
14641 fn and_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14642 crate::kernel!(
14643 #[inline(always)]
14644 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
14645 _mm256_and_si256(a.into(), b.into()).simd_into(token)
14646 }
14647 );
14648 kernel(self, a, b)
14649 }
14650 #[inline(always)]
14651 fn or_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14652 crate::kernel!(
14653 #[inline(always)]
14654 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
14655 _mm256_or_si256(a.into(), b.into()).simd_into(token)
14656 }
14657 );
14658 kernel(self, a, b)
14659 }
14660 #[inline(always)]
14661 fn xor_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14662 crate::kernel!(
14663 #[inline(always)]
14664 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
14665 _mm256_xor_si256(a.into(), b.into()).simd_into(token)
14666 }
14667 );
14668 kernel(self, a, b)
14669 }
14670 #[inline(always)]
14671 fn not_u64x4(self, a: u64x4<Self>) -> u64x4<Self> {
14672 a ^ !0
14673 }
14674 #[inline(always)]
14675 fn shl_u64x4(self, a: u64x4<Self>, shift: u32) -> u64x4<Self> {
14676 crate::kernel!(
14677 #[inline(always)]
14678 fn kernel(token: Avx2, a: u64x4<Avx2>, shift: u32) -> u64x4<Avx2> {
14679 _mm256_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
14680 }
14681 );
14682 kernel(self, a, shift)
14683 }
14684 #[inline(always)]
14685 fn shlv_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14686 crate::kernel!(
14687 #[inline(always)]
14688 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
14689 _mm256_sllv_epi64(a.into(), b.into()).simd_into(token)
14690 }
14691 );
14692 kernel(self, a, b)
14693 }
14694 #[inline(always)]
14695 fn shr_u64x4(self, a: u64x4<Self>, shift: u32) -> u64x4<Self> {
14696 crate::kernel!(
14697 #[inline(always)]
14698 fn kernel(token: Avx2, a: u64x4<Avx2>, shift: u32) -> u64x4<Avx2> {
14699 _mm256_srl_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
14700 }
14701 );
14702 kernel(self, a, shift)
14703 }
14704 #[inline(always)]
14705 fn shrv_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14706 crate::kernel!(
14707 #[inline(always)]
14708 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
14709 _mm256_srlv_epi64(a.into(), b.into()).simd_into(token)
14710 }
14711 );
14712 kernel(self, a, b)
14713 }
14714 #[inline(always)]
14715 fn max_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14716 crate::kernel!(
14717 #[inline(always)]
14718 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
14719 let a = a.into();
14720 let b = b.into();
14721 let mask = {
14722 let sign_bit = _mm256_set1_epi64x(0x8000000000000000u64.cast_signed());
14723 let lhs_signed = _mm256_xor_si256(b, sign_bit);
14724 let rhs_signed = _mm256_xor_si256(a, sign_bit);
14725 _mm256_cmpgt_epi64(lhs_signed, rhs_signed)
14726 };
14727 let result = _mm256_blendv_pd(
14728 _mm256_castsi256_pd(a),
14729 _mm256_castsi256_pd(b),
14730 _mm256_castsi256_pd(mask),
14731 );
14732 _mm256_castpd_si256(result).simd_into(token)
14733 }
14734 );
14735 kernel(self, a, b)
14736 }
14737 #[inline(always)]
14738 fn min_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14739 crate::kernel!(
14740 #[inline(always)]
14741 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
14742 let a = a.into();
14743 let b = b.into();
14744 let mask = {
14745 let sign_bit = _mm256_set1_epi64x(0x8000000000000000u64.cast_signed());
14746 let lhs_signed = _mm256_xor_si256(a, sign_bit);
14747 let rhs_signed = _mm256_xor_si256(b, sign_bit);
14748 _mm256_cmpgt_epi64(lhs_signed, rhs_signed)
14749 };
14750 let result = _mm256_blendv_pd(
14751 _mm256_castsi256_pd(a),
14752 _mm256_castsi256_pd(b),
14753 _mm256_castsi256_pd(mask),
14754 );
14755 _mm256_castpd_si256(result).simd_into(token)
14756 }
14757 );
14758 kernel(self, a, b)
14759 }
14760 #[inline(always)]
14761 fn clamp_u64x4(self, a: u64x4<Self>, b: u64x4<Self>, c: u64x4<Self>) -> u64x4<Self> {
14762 debug_assert!(
14763 b.simd_le(c).all_true(),
14764 "clamp requires min <= max for all elements"
14765 );
14766 a.max(b).min(c)
14767 }
14768 #[inline(always)]
14769 fn simd_eq_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> mask64x4<Self> {
14770 crate::kernel!(
14771 #[inline(always)]
14772 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> mask64x4<Avx2> {
14773 _mm256_cmpeq_epi64(a.into(), b.into()).simd_into(token)
14774 }
14775 );
14776 kernel(self, a, b)
14777 }
14778 #[inline(always)]
14779 fn simd_lt_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> mask64x4<Self> {
14780 crate::kernel!(
14781 #[inline(always)]
14782 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> mask64x4<Avx2> {
14783 {
14784 let sign_bit = _mm256_set1_epi64x(0x8000000000000000u64.cast_signed());
14785 let lhs_signed = _mm256_xor_si256(b.into(), sign_bit);
14786 let rhs_signed = _mm256_xor_si256(a.into(), sign_bit);
14787 _mm256_cmpgt_epi64(lhs_signed, rhs_signed)
14788 }
14789 .simd_into(token)
14790 }
14791 );
14792 kernel(self, a, b)
14793 }
14794 #[inline(always)]
14795 fn simd_le_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> mask64x4<Self> {
14796 crate::kernel!(
14797 #[inline(always)]
14798 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> mask64x4<Avx2> {
14799 _mm256_xor_si256(
14800 {
14801 let sign_bit = _mm256_set1_epi64x(0x8000000000000000u64.cast_signed());
14802 let lhs_signed = _mm256_xor_si256(a.into(), sign_bit);
14803 let rhs_signed = _mm256_xor_si256(b.into(), sign_bit);
14804 _mm256_cmpgt_epi64(lhs_signed, rhs_signed)
14805 },
14806 _mm256_set1_epi64x(-1),
14807 )
14808 .simd_into(token)
14809 }
14810 );
14811 kernel(self, a, b)
14812 }
14813 #[inline(always)]
14814 fn zip_low_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14815 crate::kernel!(
14816 #[inline(always)]
14817 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
14818 let lo = _mm256_unpacklo_epi64(a.into(), b.into());
14819 let hi = _mm256_unpackhi_epi64(a.into(), b.into());
14820 _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token)
14821 }
14822 );
14823 kernel(self, a, b)
14824 }
14825 #[inline(always)]
14826 fn zip_high_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14827 crate::kernel!(
14828 #[inline(always)]
14829 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
14830 let lo = _mm256_unpacklo_epi64(a.into(), b.into());
14831 let hi = _mm256_unpackhi_epi64(a.into(), b.into());
14832 _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token)
14833 }
14834 );
14835 kernel(self, a, b)
14836 }
14837 #[inline(always)]
14838 fn unzip_low_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14839 crate::kernel!(
14840 #[inline(always)]
14841 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
14842 let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(a.into());
14843 let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(b.into());
14844 _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token)
14845 }
14846 );
14847 kernel(self, a, b)
14848 }
14849 #[inline(always)]
14850 fn unzip_high_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
14851 crate::kernel!(
14852 #[inline(always)]
14853 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
14854 let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(a.into());
14855 let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(b.into());
14856 _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token)
14857 }
14858 );
14859 kernel(self, a, b)
14860 }
14861 #[inline(always)]
14862 fn interleave_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> (u64x4<Self>, u64x4<Self>) {
14863 crate::kernel!(
14864 #[inline(always)]
14865 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> (u64x4<Avx2>, u64x4<Avx2>) {
14866 let lo = _mm256_unpacklo_epi64(a.into(), b.into());
14867 let hi = _mm256_unpackhi_epi64(a.into(), b.into());
14868 (
14869 _mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token),
14870 _mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token),
14871 )
14872 }
14873 );
14874 kernel(self, a, b)
14875 }
14876 #[inline(always)]
14877 fn deinterleave_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> (u64x4<Self>, u64x4<Self>) {
14878 crate::kernel!(
14879 #[inline(always)]
14880 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> (u64x4<Avx2>, u64x4<Avx2>) {
14881 let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(a.into());
14882 let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(b.into());
14883 (
14884 _mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token),
14885 _mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token),
14886 )
14887 }
14888 );
14889 kernel(self, a, b)
14890 }
14891 #[inline(always)]
14892 fn select_u64x4(self, a: mask64x4<Self>, b: u64x4<Self>, c: u64x4<Self>) -> u64x4<Self> {
14893 crate::kernel!(
14894 #[inline(always)]
14895 fn kernel(
14896 token: Avx2,
14897 a: mask64x4<Avx2>,
14898 b: u64x4<Avx2>,
14899 c: u64x4<Avx2>,
14900 ) -> u64x4<Avx2> {
14901 _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
14902 }
14903 );
14904 kernel(self, a, b, c)
14905 }
14906 #[inline(always)]
14907 fn combine_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x8<Self> {
14908 u64x8 {
14909 val: crate::support::Aligned512([a.val.0, b.val.0]),
14910 simd: self,
14911 }
14912 }
14913 #[inline(always)]
14914 fn split_u64x4(self, a: u64x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
14915 crate::kernel!(
14916 #[inline(always)]
14917 fn kernel(token: Avx2, a: u64x4<Avx2>) -> (u64x2<Avx2>, u64x2<Avx2>) {
14918 (
14919 _mm256_extracti128_si256::<0>(a.into()).simd_into(token),
14920 _mm256_extracti128_si256::<1>(a.into()).simd_into(token),
14921 )
14922 }
14923 );
14924 kernel(self, a)
14925 }
14926 #[inline(always)]
14927 fn narrow_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u32x8<Self> {
14928 crate::kernel!(
14929 #[inline(always)]
14930 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u32x8<Avx2> {
14931 _mm256_permute4x64_epi64::<0xd8>(_mm256_castps_si256(_mm256_shuffle_ps::<0x88>(
14932 _mm256_castsi256_ps(a.into()),
14933 _mm256_castsi256_ps(b.into()),
14934 )))
14935 .simd_into(token)
14936 }
14937 );
14938 kernel(self, a, b)
14939 }
14940 #[inline(always)]
14941 fn saturating_narrow_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u32x8<Self> {
14942 crate::kernel!(
14943 #[inline(always)]
14944 fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u32x8<Avx2> {
14945 let zero = _mm256_setzero_si256();
14946 let ones = _mm256_cmpeq_epi64(zero, zero);
14947 let a = a.into();
14948 let b = b.into();
14949 let a_fits = _mm256_cmpeq_epi64(_mm256_srli_epi64::<32>(a), zero);
14950 let b_fits = _mm256_cmpeq_epi64(_mm256_srli_epi64::<32>(b), zero);
14951 let a = _mm256_or_si256(a, _mm256_xor_si256(a_fits, ones));
14952 let b = _mm256_or_si256(b, _mm256_xor_si256(b_fits, ones));
14953 _mm256_permute4x64_epi64::<0xd8>(_mm256_castps_si256(_mm256_shuffle_ps::<0x88>(
14954 _mm256_castsi256_ps(a),
14955 _mm256_castsi256_ps(b),
14956 )))
14957 .simd_into(token)
14958 }
14959 );
14960 kernel(self, a, b)
14961 }
14962 #[inline(always)]
14963 fn relaxed_narrow_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u32x8<Self> {
14964 debug_assert!(
14965 a.as_slice()
14966 .iter()
14967 .chain(b.as_slice())
14968 .all(|&value| { value >= u32::MIN as u64 && value <= u32::MAX as u64 }),
14969 "relaxed_narrow inputs must fit in the destination type",
14970 );
14971 self.narrow_u64x4(a, b)
14972 }
14973 #[inline(always)]
14974 fn cvt_f64_u64x4(self, a: u64x4<Self>) -> f64x4<Self> {
14975 crate::kernel!(
14976 #[inline(always)]
14977 fn kernel(token: Avx2, a: u64x4<Avx2>) -> f64x4<Avx2> {
14978 let a = a.into();
14979 let low = _mm256_blend_epi32::<0xaa>(a, _mm256_set1_epi64x(0x4330_0000_0000_0000));
14980 let high = _mm256_srli_epi64::<32>(a);
14981 let high = _mm256_xor_si256(high, _mm256_set1_epi64x(4985484787499139072i64));
14982 let high = _mm256_sub_pd(
14983 _mm256_castsi256_pd(high),
14984 _mm256_set1_pd(f64::from_bits(4985484787500187648u64)),
14985 );
14986 _mm256_add_pd(_mm256_castsi256_pd(low), high).simd_into(token)
14987 }
14988 );
14989 kernel(self, a)
14990 }
14991 #[inline(always)]
14992 fn splat_mask64x4(self, val: bool) -> mask64x4<Self> {
14993 crate::kernel!(
14994 #[inline(always)]
14995 fn kernel(token: Avx2, val: bool) -> mask64x4<Avx2> {
14996 let val: i64 = if val { !0 } else { 0 };
14997 _mm256_set1_epi64x(val).simd_into(token)
14998 }
14999 );
15000 kernel(self, val)
15001 }
15002 #[inline(always)]
15003 fn from_bitmask_mask64x4(self, bits: u64) -> mask64x4<Self> {
15004 crate::kernel!(
15005 #[inline(always)]
15006 fn kernel(token: Avx2, bits: u64) -> mask64x4<Avx2> {
15007 {
15008 let bit_lanes = _mm256_set1_epi64x(bits.cast_signed());
15009 let bit_mask = _mm256_set_epi64x(8, 4, 2, 1);
15010 _mm256_cmpeq_epi64(_mm256_and_si256(bit_lanes, bit_mask), bit_mask)
15011 }
15012 .simd_into(token)
15013 }
15014 );
15015 kernel(self, bits)
15016 }
15017 #[inline(always)]
15018 fn to_bitmask_mask64x4(self, a: mask64x4<Self>) -> u64 {
15019 crate::kernel!(
15020 #[inline(always)]
15021 fn kernel(token: Avx2, a: mask64x4<Avx2>) -> u64 {
15022 _mm256_movemask_pd(_mm256_castsi256_pd(a.into())) as u32 as u64
15023 }
15024 );
15025 kernel(self, a)
15026 }
15027 #[inline(always)]
15028 fn from_vector_mask64x4(self, vector: i64x4<Self>) -> mask64x4<Self> {
15029 mask64x4 {
15030 val: vector.val,
15031 simd: self,
15032 }
15033 }
15034 #[inline(always)]
15035 fn to_vector_mask64x4(self, a: mask64x4<Self>) -> i64x4<Self> {
15036 i64x4 {
15037 val: a.val,
15038 simd: self,
15039 }
15040 }
15041 #[inline(always)]
15042 fn set_mask64x4(self, a: &mut mask64x4<Self>, index: usize, value: bool) -> () {
15043 assert!(
15044 index < 4usize,
15045 "mask lane index {index} is out of bounds for {} lanes",
15046 4usize
15047 );
15048 let mut lanes: [i64; 4usize] = (*a).into();
15049 lanes[index] = if value { !0 } else { 0 };
15050 *a = lanes.simd_into(self);
15051 }
15052 #[inline(always)]
15053 fn rotate_elements_left_mask64x4<const OFFSET: usize>(
15054 self,
15055 a: mask64x4<Self>,
15056 ) -> mask64x4<Self> {
15057 let int = i64x4 {
15058 val: crate::transmute::checked_transmute_copy(&a.val),
15059 simd: self,
15060 };
15061 let rotated = int.rotate_elements_left::<OFFSET>();
15062 mask64x4 {
15063 val: crate::transmute::checked_transmute_copy(&rotated.val),
15064 simd: self,
15065 }
15066 }
15067 #[inline(always)]
15068 fn rotate_elements_right_mask64x4<const OFFSET: usize>(
15069 self,
15070 a: mask64x4<Self>,
15071 ) -> mask64x4<Self> {
15072 let int = i64x4 {
15073 val: crate::transmute::checked_transmute_copy(&a.val),
15074 simd: self,
15075 };
15076 let rotated = int.rotate_elements_right::<OFFSET>();
15077 mask64x4 {
15078 val: crate::transmute::checked_transmute_copy(&rotated.val),
15079 simd: self,
15080 }
15081 }
15082 #[inline(always)]
15083 fn and_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
15084 crate::kernel!(
15085 #[inline(always)]
15086 fn kernel(token: Avx2, a: mask64x4<Avx2>, b: mask64x4<Avx2>) -> mask64x4<Avx2> {
15087 _mm256_and_si256(a.into(), b.into()).simd_into(token)
15088 }
15089 );
15090 kernel(self, a, b)
15091 }
15092 #[inline(always)]
15093 fn or_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
15094 crate::kernel!(
15095 #[inline(always)]
15096 fn kernel(token: Avx2, a: mask64x4<Avx2>, b: mask64x4<Avx2>) -> mask64x4<Avx2> {
15097 _mm256_or_si256(a.into(), b.into()).simd_into(token)
15098 }
15099 );
15100 kernel(self, a, b)
15101 }
15102 #[inline(always)]
15103 fn xor_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
15104 crate::kernel!(
15105 #[inline(always)]
15106 fn kernel(token: Avx2, a: mask64x4<Avx2>, b: mask64x4<Avx2>) -> mask64x4<Avx2> {
15107 _mm256_xor_si256(a.into(), b.into()).simd_into(token)
15108 }
15109 );
15110 kernel(self, a, b)
15111 }
15112 #[inline(always)]
15113 fn not_mask64x4(self, a: mask64x4<Self>) -> mask64x4<Self> {
15114 self.xor_mask64x4(a, self.splat_mask64x4(true))
15115 }
15116 #[inline(always)]
15117 fn reverse_mask64x4(self, a: mask64x4<Self>) -> mask64x4<Self> {
15118 let lanes = i64x4 {
15119 val: crate::transmute::checked_transmute_copy(&a.val),
15120 simd: self,
15121 };
15122 let reversed = self.reverse_i64x4(lanes);
15123 mask64x4 {
15124 val: crate::transmute::checked_transmute_copy(&reversed.val),
15125 simd: self,
15126 }
15127 }
15128 #[inline(always)]
15129 fn select_mask64x4(
15130 self,
15131 a: mask64x4<Self>,
15132 b: mask64x4<Self>,
15133 c: mask64x4<Self>,
15134 ) -> mask64x4<Self> {
15135 crate::kernel!(
15136 #[inline(always)]
15137 fn kernel(
15138 token: Avx2,
15139 a: mask64x4<Avx2>,
15140 b: mask64x4<Avx2>,
15141 c: mask64x4<Avx2>,
15142 ) -> mask64x4<Avx2> {
15143 _mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
15144 }
15145 );
15146 kernel(self, a, b, c)
15147 }
15148 #[inline(always)]
15149 fn simd_eq_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
15150 crate::kernel!(
15151 #[inline(always)]
15152 fn kernel(token: Avx2, a: mask64x4<Avx2>, b: mask64x4<Avx2>) -> mask64x4<Avx2> {
15153 _mm256_cmpeq_epi64(a.into(), b.into()).simd_into(token)
15154 }
15155 );
15156 kernel(self, a, b)
15157 }
15158 #[inline(always)]
15159 fn any_true_mask64x4(self, a: mask64x4<Self>) -> bool {
15160 crate::kernel!(
15161 #[inline(always)]
15162 fn kernel(token: Avx2, a: mask64x4<Avx2>) -> bool {
15163 _mm256_movemask_pd(_mm256_castsi256_pd(a.into())) as u32 != 0
15164 }
15165 );
15166 kernel(self, a)
15167 }
15168 #[inline(always)]
15169 fn all_true_mask64x4(self, a: mask64x4<Self>) -> bool {
15170 crate::kernel!(
15171 #[inline(always)]
15172 fn kernel(token: Avx2, a: mask64x4<Avx2>) -> bool {
15173 _mm256_movemask_pd(_mm256_castsi256_pd(a.into())) as u32 == 0b1111
15174 }
15175 );
15176 kernel(self, a)
15177 }
15178 #[inline(always)]
15179 fn any_false_mask64x4(self, a: mask64x4<Self>) -> bool {
15180 crate::kernel!(
15181 #[inline(always)]
15182 fn kernel(token: Avx2, a: mask64x4<Avx2>) -> bool {
15183 _mm256_movemask_pd(_mm256_castsi256_pd(a.into())) as u32 != 0b1111
15184 }
15185 );
15186 kernel(self, a)
15187 }
15188 #[inline(always)]
15189 fn all_false_mask64x4(self, a: mask64x4<Self>) -> bool {
15190 crate::kernel!(
15191 #[inline(always)]
15192 fn kernel(token: Avx2, a: mask64x4<Avx2>) -> bool {
15193 _mm256_movemask_pd(_mm256_castsi256_pd(a.into())) as u32 == 0
15194 }
15195 );
15196 kernel(self, a)
15197 }
15198 #[inline(always)]
15199 fn combine_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x8<Self> {
15200 mask64x8 {
15201 val: crate::support::Aligned512([a.val.0, b.val.0]),
15202 simd: self,
15203 }
15204 }
15205 #[inline(always)]
15206 fn split_mask64x4(self, a: mask64x4<Self>) -> (mask64x2<Self>, mask64x2<Self>) {
15207 crate::kernel!(
15208 #[inline(always)]
15209 fn kernel(token: Avx2, a: mask64x4<Avx2>) -> (mask64x2<Avx2>, mask64x2<Avx2>) {
15210 (
15211 _mm256_extracti128_si256::<0>(a.into()).simd_into(token),
15212 _mm256_extracti128_si256::<1>(a.into()).simd_into(token),
15213 )
15214 }
15215 );
15216 kernel(self, a)
15217 }
15218 #[inline(always)]
15219 fn narrow_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask32x8<Self> {
15220 crate::kernel!(
15221 #[inline(always)]
15222 fn kernel(token: Avx2, a: mask64x4<Avx2>, b: mask64x4<Avx2>) -> mask32x8<Avx2> {
15223 _mm256_permute4x64_epi64::<0xd8>(_mm256_castps_si256(_mm256_shuffle_ps::<0x88>(
15224 _mm256_castsi256_ps(a.into()),
15225 _mm256_castsi256_ps(b.into()),
15226 )))
15227 .simd_into(token)
15228 }
15229 );
15230 kernel(self, a, b)
15231 }
15232 #[inline(always)]
15233 fn slide_f32x16<const SHIFT: usize>(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
15234 if SHIFT >= 16usize {
15235 return b;
15236 }
15237 let result = cross_block_alignr_256x2(
15238 self,
15239 Bytes::to_bytes(b).val.0,
15240 Bytes::to_bytes(a).val.0,
15241 SHIFT * 4usize,
15242 );
15243 Bytes::from_bytes(u8x64 {
15244 val: crate::support::Aligned512(result),
15245 simd: self,
15246 })
15247 }
15248 #[inline(always)]
15249 fn compress_f32x16(self, values: f32x16<Self>, mask: mask32x16<Self>) -> f32x16<Self> {
15250 crate::kernel!(
15251 #[inline(always)]
15252 fn kernel(token: Avx2, values: f32x16<Avx2>, mask: mask32x16<Avx2>) -> f32x16<Avx2> {
15253 let (low_mask, high_mask) = token.split_mask32x16(mask);
15254 let low_count = low_mask.to_bitmask().count_ones() as usize;
15255 let (low, high) = values.split();
15256 let bits = mask.to_bitmask();
15257 if bits == 0 {
15258 return f32x16::splat(token, 0.0);
15259 }
15260 if bits == 65535 {
15261 return values;
15262 }
15263 let low = low.compress(low_mask);
15264 let high = high.compress(high_mask);
15265 let control: __m256i = crate::transmute::checked_transmute_copy(
15266 &crate::support::compact_256::SHIFTS[low_count],
15267 );
15268 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
15269 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
15270 let shifted = _mm256_permutevar8x32_epi32(high_bits, control);
15271 let result_low = _mm256_blendv_epi8(shifted, low_bits, control);
15272 let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control));
15273 f32x16 {
15274 val: crate::transmute::checked_transmute_copy(&[result_low, result_high]),
15275 simd: token,
15276 }
15277 }
15278 );
15279 kernel(self, values, mask)
15280 }
15281 #[inline(always)]
15282 fn compress_merge_f32x16(
15283 self,
15284 values: f32x16<Self>,
15285 mask: mask32x16<Self>,
15286 merge: f32x16<Self>,
15287 ) -> f32x16<Self> {
15288 crate::kernel!(
15289 #[inline(always)]
15290 fn kernel(
15291 token: Avx2,
15292 values: f32x16<Avx2>,
15293 mask: mask32x16<Avx2>,
15294 merge: f32x16<Avx2>,
15295 ) -> f32x16<Avx2> {
15296 let (low_mask, high_mask) = token.split_mask32x16(mask);
15297 let low_count = low_mask.to_bitmask().count_ones() as usize;
15298 let (low, high) = values.split();
15299 let bits = mask.to_bitmask();
15300 if bits == 0 {
15301 return merge;
15302 }
15303 if bits == 65535 {
15304 return values;
15305 }
15306 let low = low.compress(low_mask);
15307 let high = high.compress(high_mask);
15308 let control: __m256i = crate::transmute::checked_transmute_copy(
15309 &crate::support::compact_256::SHIFTS[low_count],
15310 );
15311 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
15312 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
15313 let shifted = _mm256_permutevar8x32_epi32(high_bits, control);
15314 let result_low = _mm256_blendv_epi8(shifted, low_bits, control);
15315 let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control));
15316 let compressed = f32x16 {
15317 val: crate::transmute::checked_transmute_copy(&[result_low, result_high]),
15318 simd: token,
15319 };
15320 let count = low_count + high_mask.to_bitmask().count_ones() as usize;
15321 let prefix = mask32x16::from_bitmask(token, (1u64 << count) - 1);
15322 token.select_f32x16(prefix, compressed, merge)
15323 }
15324 );
15325 kernel(self, values, mask, merge)
15326 }
15327 #[inline(always)]
15328 fn expand_f32x16(self, values: f32x16<Self>, mask: mask32x16<Self>) -> f32x16<Self> {
15329 crate::kernel!(
15330 #[inline(always)]
15331 fn kernel(token: Avx2, values: f32x16<Avx2>, mask: mask32x16<Avx2>) -> f32x16<Avx2> {
15332 let (low_mask, high_mask) = token.split_mask32x16(mask);
15333 let low_count = low_mask.to_bitmask().count_ones() as usize;
15334 let (low, high) = values.split();
15335 let control: __m256i = crate::transmute::checked_transmute_copy(
15336 &crate::support::compact_256::SHIFTS[8 - (low_count)],
15337 );
15338 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
15339 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
15340 let from_low = _mm256_permutevar8x32_epi32(low_bits, control);
15341 let from_high = _mm256_permutevar8x32_epi32(high_bits, control);
15342 let packed = _mm256_blendv_epi8(from_high, from_low, control);
15343 let high = f32x8 {
15344 val: crate::transmute::checked_transmute_copy(&packed),
15345 simd: token,
15346 };
15347 low.expand(low_mask).combine(high.expand(high_mask))
15348 }
15349 );
15350 kernel(self, values, mask)
15351 }
15352 #[inline(always)]
15353 fn expand_merge_f32x16(
15354 self,
15355 values: f32x16<Self>,
15356 mask: mask32x16<Self>,
15357 merge: f32x16<Self>,
15358 ) -> f32x16<Self> {
15359 crate::kernel!(
15360 #[inline(always)]
15361 fn kernel(
15362 token: Avx2,
15363 values: f32x16<Avx2>,
15364 mask: mask32x16<Avx2>,
15365 merge: f32x16<Avx2>,
15366 ) -> f32x16<Avx2> {
15367 let (low_mask, high_mask) = token.split_mask32x16(mask);
15368 let low_count = low_mask.to_bitmask().count_ones() as usize;
15369 let (low, high) = values.split();
15370 let control: __m256i = crate::transmute::checked_transmute_copy(
15371 &crate::support::compact_256::SHIFTS[8 - (low_count)],
15372 );
15373 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
15374 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
15375 let from_low = _mm256_permutevar8x32_epi32(low_bits, control);
15376 let from_high = _mm256_permutevar8x32_epi32(high_bits, control);
15377 let packed = _mm256_blendv_epi8(from_high, from_low, control);
15378 let high = f32x8 {
15379 val: crate::transmute::checked_transmute_copy(&packed),
15380 simd: token,
15381 };
15382 let (low_merge, high_merge) = merge.split();
15383 low.expand_merge(low_mask, low_merge)
15384 .combine(high.expand_merge(high_mask, high_merge))
15385 }
15386 );
15387 kernel(self, values, mask, merge)
15388 }
15389 #[inline(always)]
15390 fn simd_ne_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> mask32x16<Self> {
15391 let (a0, a1) = self.split_f32x16(a);
15392 let (b0, b1) = self.split_f32x16(b);
15393 self.combine_mask32x8(self.simd_ne_f32x8(a0, b0), self.simd_ne_f32x8(a1, b1))
15394 }
15395 #[inline(always)]
15396 fn is_nan_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
15397 a.simd_ne(a)
15398 }
15399 #[inline(always)]
15400 fn is_infinite_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
15401 a.abs().simd_eq(f32::INFINITY)
15402 }
15403 #[inline(always)]
15404 fn is_finite_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
15405 a.abs().simd_lt(f32::INFINITY)
15406 }
15407 #[inline(always)]
15408 fn is_subnormal_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
15409 let i: i32x16<Self> = a.bitcast();
15410 (i & 0x7F80_0000_i32).simd_eq(0) & (i & 0x007F_FFFF_i32).simd_ne(0)
15411 }
15412 #[inline(always)]
15413 fn is_normal_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
15414 let i: i32x16<Self> = a.bitcast();
15415 let exp = i & 0x7F80_0000_i32;
15416 exp.simd_ne(0) & exp.simd_ne(0x7F80_0000_i32)
15417 }
15418 #[inline(always)]
15419 fn is_sign_positive_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
15420 let i: i32x16<Self> = a.bitcast();
15421 (i & i32::MIN).simd_eq(0)
15422 }
15423 #[inline(always)]
15424 fn is_sign_negative_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
15425 !a.is_sign_positive()
15426 }
15427 #[inline(always)]
15428 fn split_f32x16(self, a: f32x16<Self>) -> (f32x8<Self>, f32x8<Self>) {
15429 (
15430 f32x8 {
15431 val: crate::support::Aligned256(a.val.0[0]),
15432 simd: self,
15433 },
15434 f32x8 {
15435 val: crate::support::Aligned256(a.val.0[1]),
15436 simd: self,
15437 },
15438 )
15439 }
15440 #[inline(always)]
15441 fn slide_i8x64<const SHIFT: usize>(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
15442 if SHIFT >= 64usize {
15443 return b;
15444 }
15445 let result = cross_block_alignr_256x2(
15446 self,
15447 Bytes::to_bytes(b).val.0,
15448 Bytes::to_bytes(a).val.0,
15449 SHIFT,
15450 );
15451 Bytes::from_bytes(u8x64 {
15452 val: crate::support::Aligned512(result),
15453 simd: self,
15454 })
15455 }
15456 #[inline(always)]
15457 fn compress_i8x64(self, values: i8x64<Self>, mask: mask8x64<Self>) -> i8x64<Self> {
15458 let mask = mask8x64 {
15459 val: crate::transmute::checked_transmute_copy(&mask.val),
15460 simd: self,
15461 };
15462 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
15463 }
15464 #[inline(always)]
15465 fn compress_merge_i8x64(
15466 self,
15467 values: i8x64<Self>,
15468 mask: mask8x64<Self>,
15469 merge: i8x64<Self>,
15470 ) -> i8x64<Self> {
15471 let mask = mask8x64 {
15472 val: crate::transmute::checked_transmute_copy(&mask.val),
15473 simd: self,
15474 };
15475 Bytes::from_bytes(self.compress_merge_u8x64(
15476 Bytes::to_bytes(values),
15477 mask,
15478 Bytes::to_bytes(merge),
15479 ))
15480 }
15481 #[inline(always)]
15482 fn expand_i8x64(self, values: i8x64<Self>, mask: mask8x64<Self>) -> i8x64<Self> {
15483 let mask = mask8x64 {
15484 val: crate::transmute::checked_transmute_copy(&mask.val),
15485 simd: self,
15486 };
15487 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
15488 }
15489 #[inline(always)]
15490 fn expand_merge_i8x64(
15491 self,
15492 values: i8x64<Self>,
15493 mask: mask8x64<Self>,
15494 merge: i8x64<Self>,
15495 ) -> i8x64<Self> {
15496 let mask = mask8x64 {
15497 val: crate::transmute::checked_transmute_copy(&mask.val),
15498 simd: self,
15499 };
15500 Bytes::from_bytes(self.expand_merge_u8x64(
15501 Bytes::to_bytes(values),
15502 mask,
15503 Bytes::to_bytes(merge),
15504 ))
15505 }
15506 #[inline(always)]
15507 fn split_i8x64(self, a: i8x64<Self>) -> (i8x32<Self>, i8x32<Self>) {
15508 (
15509 i8x32 {
15510 val: crate::support::Aligned256(a.val.0[0]),
15511 simd: self,
15512 },
15513 i8x32 {
15514 val: crate::support::Aligned256(a.val.0[1]),
15515 simd: self,
15516 },
15517 )
15518 }
15519 #[inline(always)]
15520 fn slide_u8x64<const SHIFT: usize>(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
15521 if SHIFT >= 64usize {
15522 return b;
15523 }
15524 let result = cross_block_alignr_256x2(
15525 self,
15526 Bytes::to_bytes(b).val.0,
15527 Bytes::to_bytes(a).val.0,
15528 SHIFT,
15529 );
15530 Bytes::from_bytes(u8x64 {
15531 val: crate::support::Aligned512(result),
15532 simd: self,
15533 })
15534 }
15535 #[inline(always)]
15536 fn swizzle_dyn_u8x64(self, a: u8x64<Self>, indices: u8x64<Self>) -> u8x64<Self> {
15537 self.swizzle_dyn_precise_u8x64(a, indices)
15538 }
15539 #[inline(always)]
15540 fn swizzle_dyn_precise_u8x64(self, a: u8x64<Self>, indices: u8x64<Self>) -> u8x64<Self> {
15541 crate::kernel!(
15542 #[inline(always)]
15543 fn kernel(token: Avx2, a: u8x64<Avx2>, indices: u8x64<Avx2>) -> u8x64<Avx2> {
15544 let bytes = Bytes::to_bytes(a);
15545 let (table_low, table_high) = token.split_u8x64(bytes);
15546 let (indices_low, indices_high) = token.split_u8x64(indices);
15547 let high_table_offset = token.splat_u8x32(32);
15548 let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low);
15549 let output_low_from_high = token.swizzle_dyn_precise_u8x32(
15550 table_high,
15551 token.sub_u8x32(indices_low, high_table_offset),
15552 );
15553 let output_low = token.or_u8x32(output_low_from_low, output_low_from_high);
15554 let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high);
15555 let output_high_from_high = token.swizzle_dyn_precise_u8x32(
15556 table_high,
15557 token.sub_u8x32(indices_high, high_table_offset),
15558 );
15559 let output_high = token.or_u8x32(output_high_from_low, output_high_from_high);
15560 let result_bytes = token.combine_u8x32(output_low, output_high);
15561 Bytes::from_bytes(result_bytes)
15562 }
15563 );
15564 kernel(self, a, indices)
15565 }
15566 #[inline(always)]
15567 fn concat_swizzle_dyn_u8x64(
15568 self,
15569 a: u8x64<Self>,
15570 b: u8x64<Self>,
15571 indices: u8x64<Self>,
15572 ) -> u8x64<Self> {
15573 self.concat_swizzle_dyn_precise_u8x64(a, b, indices)
15574 }
15575 #[inline(always)]
15576 fn concat_swizzle_dyn_precise_u8x64(
15577 self,
15578 a: u8x64<Self>,
15579 b: u8x64<Self>,
15580 indices: u8x64<Self>,
15581 ) -> u8x64<Self> {
15582 crate::kernel!(
15583 #[inline(always)]
15584 fn kernel(
15585 token: Avx2,
15586 a: u8x64<Avx2>,
15587 b: u8x64<Avx2>,
15588 indices: u8x64<Avx2>,
15589 ) -> u8x64<Avx2> {
15590 let a_bytes = Bytes::to_bytes(a).val.0;
15591 let b_bytes = Bytes::to_bytes(b).val.0;
15592 let indices_bytes = indices.val.0;
15593 let result_low = {
15594 let a = a_bytes;
15595 let b = b_bytes;
15596 let indices = indices_bytes[0];
15597 let table_bias = _mm256_set1_epi8(64);
15598 let a_control = _mm256_adds_epu8(indices, table_bias);
15599 let b_control =
15600 _mm256_adds_epu8(_mm256_xor_si256(indices, table_bias), table_bias);
15601 let select_high = _mm256_slli_epi16::<2>(indices);
15602 let flip_high_lane =
15603 _mm256_set_m128i(_mm_set1_epi8(i8::MIN), _mm_setzero_si128());
15604 let select_remote =
15605 _mm256_xor_si256(_mm256_slli_epi16::<3>(indices), flip_high_lane);
15606 let local_low = _mm256_or_si256(
15607 _mm256_shuffle_epi8(a[0], a_control),
15608 _mm256_shuffle_epi8(b[0], b_control),
15609 );
15610 let local_high = _mm256_or_si256(
15611 _mm256_shuffle_epi8(a[1], a_control),
15612 _mm256_shuffle_epi8(b[1], b_control),
15613 );
15614 let local = _mm256_blendv_epi8(local_low, local_high, select_high);
15615 let remote_low = _mm256_or_si256(
15616 _mm256_shuffle_epi8(
15617 _mm256_permute2x128_si256::<0x01>(a[0], a[0]),
15618 a_control,
15619 ),
15620 _mm256_shuffle_epi8(
15621 _mm256_permute2x128_si256::<0x01>(b[0], b[0]),
15622 b_control,
15623 ),
15624 );
15625 let remote_high = _mm256_or_si256(
15626 _mm256_shuffle_epi8(
15627 _mm256_permute2x128_si256::<0x01>(a[1], a[1]),
15628 a_control,
15629 ),
15630 _mm256_shuffle_epi8(
15631 _mm256_permute2x128_si256::<0x01>(b[1], b[1]),
15632 b_control,
15633 ),
15634 );
15635 let remote = _mm256_blendv_epi8(remote_low, remote_high, select_high);
15636 _mm256_blendv_epi8(local, remote, select_remote)
15637 };
15638 let result_high = {
15639 let a = a_bytes;
15640 let b = b_bytes;
15641 let indices = indices_bytes[1];
15642 let table_bias = _mm256_set1_epi8(64);
15643 let a_control = _mm256_adds_epu8(indices, table_bias);
15644 let b_control =
15645 _mm256_adds_epu8(_mm256_xor_si256(indices, table_bias), table_bias);
15646 let select_high = _mm256_slli_epi16::<2>(indices);
15647 let flip_high_lane =
15648 _mm256_set_m128i(_mm_set1_epi8(i8::MIN), _mm_setzero_si128());
15649 let select_remote =
15650 _mm256_xor_si256(_mm256_slli_epi16::<3>(indices), flip_high_lane);
15651 let local_low = _mm256_or_si256(
15652 _mm256_shuffle_epi8(a[0], a_control),
15653 _mm256_shuffle_epi8(b[0], b_control),
15654 );
15655 let local_high = _mm256_or_si256(
15656 _mm256_shuffle_epi8(a[1], a_control),
15657 _mm256_shuffle_epi8(b[1], b_control),
15658 );
15659 let local = _mm256_blendv_epi8(local_low, local_high, select_high);
15660 let remote_low = _mm256_or_si256(
15661 _mm256_shuffle_epi8(
15662 _mm256_permute2x128_si256::<0x01>(a[0], a[0]),
15663 a_control,
15664 ),
15665 _mm256_shuffle_epi8(
15666 _mm256_permute2x128_si256::<0x01>(b[0], b[0]),
15667 b_control,
15668 ),
15669 );
15670 let remote_high = _mm256_or_si256(
15671 _mm256_shuffle_epi8(
15672 _mm256_permute2x128_si256::<0x01>(a[1], a[1]),
15673 a_control,
15674 ),
15675 _mm256_shuffle_epi8(
15676 _mm256_permute2x128_si256::<0x01>(b[1], b[1]),
15677 b_control,
15678 ),
15679 );
15680 let remote = _mm256_blendv_epi8(remote_low, remote_high, select_high);
15681 _mm256_blendv_epi8(local, remote, select_remote)
15682 };
15683 let result_bytes = u8x64 {
15684 val: crate::support::Aligned512([result_low, result_high]),
15685 simd: token,
15686 };
15687 Bytes::from_bytes(result_bytes)
15688 }
15689 );
15690 kernel(self, a, b, indices)
15691 }
15692 #[inline(always)]
15693 fn compress_u8x64(self, values: u8x64<Self>, mask: mask8x64<Self>) -> u8x64<Self> {
15694 crate::kernel!(
15695 #[inline(always)]
15696 fn kernel(token: Avx2, values: u8x64<Avx2>, mask: mask8x64<Avx2>) -> u8x64<Avx2> {
15697 let mask_bits = token.to_bitmask_mask8x64(mask);
15698 if mask_bits == 0 || mask_bits == u64::MAX {
15699 return if mask_bits == 0 {
15700 u8x64::splat(token, 0)
15701 } else {
15702 values
15703 };
15704 }
15705 if mask_bits.is_power_of_two() {
15706 let selected = values.as_array()[mask_bits.trailing_zeros() as usize];
15707 return {
15708 let first = _mm256_set_epi64x(0, 0, 0, i64::from(selected));
15709 u8x64 {
15710 val: crate::support::Aligned512([first, _mm256_setzero_si256()]),
15711 simd: token,
15712 }
15713 };
15714 }
15715 let block_bits_0 = (mask_bits & 0xffff) as usize;
15716 let low_mask_0 = (block_bits_0) & 0xff;
15717 let high_mask_0 = (block_bits_0) >> 8;
15718 let low_control_0 = crate::support::COMPRESS_8_CONTROLS[low_mask_0];
15719 let high_control_0 = crate::support::COMPRESS_8_CONTROLS[high_mask_0];
15720 let high_control_0 = high_control_0 | 0x0808_0808_0808_0808;
15721 let control_0 =
15722 _mm_set_epi64x(high_control_0.cast_signed(), low_control_0.cast_signed());
15723 let low_count_0 = low_mask_0.count_ones() as usize;
15724 let block_bits_1 = (mask_bits >> 16 & 0xffff) as usize;
15725 let low_mask_1 = (block_bits_1) & 0xff;
15726 let high_mask_1 = (block_bits_1) >> 8;
15727 let low_control_1 = crate::support::COMPRESS_8_CONTROLS[low_mask_1];
15728 let high_control_1 = crate::support::COMPRESS_8_CONTROLS[high_mask_1];
15729 let high_control_1 = high_control_1 | 0x0808_0808_0808_0808;
15730 let control_1 =
15731 _mm_set_epi64x(high_control_1.cast_signed(), low_control_1.cast_signed());
15732 let low_count_1 = low_mask_1.count_ones() as usize;
15733 let control = _mm256_set_m128i(control_1, control_0);
15734 let compacted = _mm256_shuffle_epi8(values.val.0[0], control);
15735 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_0];
15736 let splice_0 = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
15737 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_1];
15738 let splice_1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
15739 let splice = _mm256_set_m128i(splice_1, splice_0);
15740 let compressed = _mm256_shuffle_epi8(compacted, splice);
15741 let splice = crate::transmute::checked_transmute_copy(
15742 &crate::support::COMPACT_16_SPLICE_CONTROLS[block_bits_0.count_ones() as usize],
15743 );
15744 let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed);
15745 let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed);
15746 let compressed_0 =
15747 _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice));
15748 let block_bits_2 = (mask_bits >> 32 & 0xffff) as usize;
15749 let low_mask_2 = (block_bits_2) & 0xff;
15750 let high_mask_2 = (block_bits_2) >> 8;
15751 let low_control_2 = crate::support::COMPRESS_8_CONTROLS[low_mask_2];
15752 let high_control_2 = crate::support::COMPRESS_8_CONTROLS[high_mask_2];
15753 let high_control_2 = high_control_2 | 0x0808_0808_0808_0808;
15754 let control_2 =
15755 _mm_set_epi64x(high_control_2.cast_signed(), low_control_2.cast_signed());
15756 let low_count_2 = low_mask_2.count_ones() as usize;
15757 let block_bits_3 = (mask_bits >> 48 & 0xffff) as usize;
15758 let low_mask_3 = (block_bits_3) & 0xff;
15759 let high_mask_3 = (block_bits_3) >> 8;
15760 let low_control_3 = crate::support::COMPRESS_8_CONTROLS[low_mask_3];
15761 let high_control_3 = crate::support::COMPRESS_8_CONTROLS[high_mask_3];
15762 let high_control_3 = high_control_3 | 0x0808_0808_0808_0808;
15763 let control_3 =
15764 _mm_set_epi64x(high_control_3.cast_signed(), low_control_3.cast_signed());
15765 let low_count_3 = low_mask_3.count_ones() as usize;
15766 let control = _mm256_set_m128i(control_3, control_2);
15767 let compacted = _mm256_shuffle_epi8(values.val.0[1], control);
15768 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_2];
15769 let splice_2 = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
15770 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_3];
15771 let splice_3 = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
15772 let splice = _mm256_set_m128i(splice_3, splice_2);
15773 let compressed = _mm256_shuffle_epi8(compacted, splice);
15774 let splice = crate::transmute::checked_transmute_copy(
15775 &crate::support::COMPACT_16_SPLICE_CONTROLS[block_bits_2.count_ones() as usize],
15776 );
15777 let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed);
15778 let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed);
15779 let compressed_1 =
15780 _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice));
15781 let first_count = (block_bits_0.count_ones() + block_bits_1.count_ones()) as usize;
15782 let controls = &crate::support::COMPACT_32_STITCH_CONTROLS[first_count];
15783 let left_same = crate::transmute::checked_transmute_copy(&controls.left_same);
15784 let left_cross = crate::transmute::checked_transmute_copy(&controls.left_cross);
15785 let right_same = crate::transmute::checked_transmute_copy(&controls.right_same);
15786 let right_cross = crate::transmute::checked_transmute_copy(&controls.right_cross);
15787 let low_to_high = _mm256_permute2x128_si256::<0x08>(compressed_1, compressed_1);
15788 let shifted_left = _mm256_or_si256(
15789 _mm256_shuffle_epi8(compressed_1, left_same),
15790 _mm256_shuffle_epi8(low_to_high, left_cross),
15791 );
15792 let compressed_0 = _mm256_or_si256(compressed_0, shifted_left);
15793 let high_to_low = _mm256_permute2x128_si256::<0x81>(compressed_1, compressed_1);
15794 let compressed_1 = _mm256_or_si256(
15795 _mm256_shuffle_epi8(compressed_1, right_same),
15796 _mm256_shuffle_epi8(high_to_low, right_cross),
15797 );
15798 let result_0 = compressed_0;
15799 let result_1 = compressed_1;
15800 u8x64 {
15801 val: crate::support::Aligned512([result_0, result_1]),
15802 simd: token,
15803 }
15804 }
15805 );
15806 kernel(self, values, mask)
15807 }
15808 #[inline(always)]
15809 fn compress_merge_u8x64(
15810 self,
15811 values: u8x64<Self>,
15812 mask: mask8x64<Self>,
15813 merge: u8x64<Self>,
15814 ) -> u8x64<Self> {
15815 crate::kernel!(
15816 #[inline(always)]
15817 fn kernel(
15818 token: Avx2,
15819 values: u8x64<Avx2>,
15820 mask: mask8x64<Avx2>,
15821 merge: u8x64<Avx2>,
15822 ) -> u8x64<Avx2> {
15823 let mask_bits = token.to_bitmask_mask8x64(mask);
15824 if mask_bits == 0 || mask_bits == u64::MAX {
15825 return if mask_bits == 0 { merge } else { values };
15826 }
15827 if mask_bits.is_power_of_two() {
15828 let selected = values.as_array()[mask_bits.trailing_zeros() as usize];
15829 return {
15830 let low = _mm_insert_epi8::<0>(
15831 _mm256_castsi256_si128(merge.val.0[0]),
15832 i32::from(selected),
15833 );
15834 let first = _mm256_inserti128_si256::<0>(merge.val.0[0], low);
15835 u8x64 {
15836 val: crate::support::Aligned512([first, merge.val.0[1]]),
15837 simd: token,
15838 }
15839 };
15840 }
15841 let block_bits_0 = (mask_bits & 0xffff) as usize;
15842 let low_mask_0 = (block_bits_0) & 0xff;
15843 let high_mask_0 = (block_bits_0) >> 8;
15844 let low_control_0 = crate::support::COMPRESS_8_CONTROLS[low_mask_0];
15845 let high_control_0 = crate::support::COMPRESS_8_CONTROLS[high_mask_0];
15846 let high_control_0 = high_control_0 | 0x0808_0808_0808_0808;
15847 let control_0 =
15848 _mm_set_epi64x(high_control_0.cast_signed(), low_control_0.cast_signed());
15849 let low_count_0 = low_mask_0.count_ones() as usize;
15850 let block_bits_1 = (mask_bits >> 16 & 0xffff) as usize;
15851 let low_mask_1 = (block_bits_1) & 0xff;
15852 let high_mask_1 = (block_bits_1) >> 8;
15853 let low_control_1 = crate::support::COMPRESS_8_CONTROLS[low_mask_1];
15854 let high_control_1 = crate::support::COMPRESS_8_CONTROLS[high_mask_1];
15855 let high_control_1 = high_control_1 | 0x0808_0808_0808_0808;
15856 let control_1 =
15857 _mm_set_epi64x(high_control_1.cast_signed(), low_control_1.cast_signed());
15858 let low_count_1 = low_mask_1.count_ones() as usize;
15859 let control = _mm256_set_m128i(control_1, control_0);
15860 let compacted = _mm256_shuffle_epi8(values.val.0[0], control);
15861 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_0];
15862 let splice_0 = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
15863 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_1];
15864 let splice_1 = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
15865 let splice = _mm256_set_m128i(splice_1, splice_0);
15866 let compressed = _mm256_shuffle_epi8(compacted, splice);
15867 let splice = crate::transmute::checked_transmute_copy(
15868 &crate::support::COMPACT_16_SPLICE_CONTROLS[block_bits_0.count_ones() as usize],
15869 );
15870 let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed);
15871 let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed);
15872 let compressed_0 =
15873 _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice));
15874 let block_bits_2 = (mask_bits >> 32 & 0xffff) as usize;
15875 let low_mask_2 = (block_bits_2) & 0xff;
15876 let high_mask_2 = (block_bits_2) >> 8;
15877 let low_control_2 = crate::support::COMPRESS_8_CONTROLS[low_mask_2];
15878 let high_control_2 = crate::support::COMPRESS_8_CONTROLS[high_mask_2];
15879 let high_control_2 = high_control_2 | 0x0808_0808_0808_0808;
15880 let control_2 =
15881 _mm_set_epi64x(high_control_2.cast_signed(), low_control_2.cast_signed());
15882 let low_count_2 = low_mask_2.count_ones() as usize;
15883 let block_bits_3 = (mask_bits >> 48 & 0xffff) as usize;
15884 let low_mask_3 = (block_bits_3) & 0xff;
15885 let high_mask_3 = (block_bits_3) >> 8;
15886 let low_control_3 = crate::support::COMPRESS_8_CONTROLS[low_mask_3];
15887 let high_control_3 = crate::support::COMPRESS_8_CONTROLS[high_mask_3];
15888 let high_control_3 = high_control_3 | 0x0808_0808_0808_0808;
15889 let control_3 =
15890 _mm_set_epi64x(high_control_3.cast_signed(), low_control_3.cast_signed());
15891 let low_count_3 = low_mask_3.count_ones() as usize;
15892 let control = _mm256_set_m128i(control_3, control_2);
15893 let compacted = _mm256_shuffle_epi8(values.val.0[1], control);
15894 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_2];
15895 let splice_2 = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
15896 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count_3];
15897 let splice_3 = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
15898 let splice = _mm256_set_m128i(splice_3, splice_2);
15899 let compressed = _mm256_shuffle_epi8(compacted, splice);
15900 let splice = crate::transmute::checked_transmute_copy(
15901 &crate::support::COMPACT_16_SPLICE_CONTROLS[block_bits_2.count_ones() as usize],
15902 );
15903 let compressed_low = _mm256_permute2x128_si256::<0x80>(compressed, compressed);
15904 let compressed_high = _mm256_permute2x128_si256::<0x11>(compressed, compressed);
15905 let compressed_1 =
15906 _mm256_or_si256(compressed_low, _mm256_shuffle_epi8(compressed_high, splice));
15907 let first_count = (block_bits_0.count_ones() + block_bits_1.count_ones()) as usize;
15908 let controls = &crate::support::COMPACT_32_STITCH_CONTROLS[first_count];
15909 let left_same = crate::transmute::checked_transmute_copy(&controls.left_same);
15910 let left_cross = crate::transmute::checked_transmute_copy(&controls.left_cross);
15911 let right_same = crate::transmute::checked_transmute_copy(&controls.right_same);
15912 let right_cross = crate::transmute::checked_transmute_copy(&controls.right_cross);
15913 let low_to_high = _mm256_permute2x128_si256::<0x08>(compressed_1, compressed_1);
15914 let shifted_left = _mm256_or_si256(
15915 _mm256_shuffle_epi8(compressed_1, left_same),
15916 _mm256_shuffle_epi8(low_to_high, left_cross),
15917 );
15918 let compressed_0 = _mm256_or_si256(compressed_0, shifted_left);
15919 let high_to_low = _mm256_permute2x128_si256::<0x81>(compressed_1, compressed_1);
15920 let compressed_1 = _mm256_or_si256(
15921 _mm256_shuffle_epi8(compressed_1, right_same),
15922 _mm256_shuffle_epi8(high_to_low, right_cross),
15923 );
15924 let output_lane = mask_bits.count_ones() as usize;
15925 let prefix = crate::transmute::checked_transmute_copy(
15926 &crate::support::COMPACT_PREFIX_MASKS[output_lane.min(32)],
15927 );
15928 let result_0 = _mm256_blendv_epi8(merge.val.0[0], compressed_0, prefix);
15929 let prefix = crate::transmute::checked_transmute_copy(
15930 &crate::support::COMPACT_PREFIX_MASKS[output_lane.saturating_sub(32).min(32)],
15931 );
15932 let result_1 = _mm256_blendv_epi8(merge.val.0[1], compressed_1, prefix);
15933 u8x64 {
15934 val: crate::support::Aligned512([result_0, result_1]),
15935 simd: token,
15936 }
15937 }
15938 );
15939 kernel(self, values, mask, merge)
15940 }
15941 #[inline(always)]
15942 fn expand_u8x64(self, values: u8x64<Self>, mask: mask8x64<Self>) -> u8x64<Self> {
15943 crate::kernel!(
15944 #[inline(always)]
15945 fn kernel(token: Avx2, values: u8x64<Avx2>, mask: mask8x64<Avx2>) -> u8x64<Avx2> {
15946 let input = <[u8; 64]>::from(values);
15947 let mut output = [0u8; 64];
15948 let mut input_lane = 0;
15949 let block_bits = _mm_movemask_epi8(_mm256_castsi256_si128(mask.val.0[0])) as usize;
15950 let low_mask = (block_bits) & 0xff;
15951 let high_mask = (block_bits) >> 8;
15952 let low_count = low_mask.count_ones() as u64;
15953 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
15954 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
15955 let high_base = low_count * 0x0101_0101_0101_0101;
15956 let high_control = high_control + high_base;
15957 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
15958 let packed = crate::transmute::checked_transmute_copy(
15959 input[input_lane..].first_chunk::<16>().unwrap(),
15960 );
15961 let expanded = _mm_shuffle_epi8(packed, control);
15962 let result = expanded;
15963 crate::transmute::checked_transmute_store(
15964 result,
15965 output[0..].first_chunk_mut::<16>().unwrap(),
15966 );
15967 input_lane += block_bits.count_ones() as usize;
15968 let block_bits =
15969 _mm_movemask_epi8(_mm256_extracti128_si256::<1>(mask.val.0[0])) as usize;
15970 let low_mask = (block_bits) & 0xff;
15971 let high_mask = (block_bits) >> 8;
15972 let low_count = low_mask.count_ones() as u64;
15973 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
15974 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
15975 let high_base = low_count * 0x0101_0101_0101_0101;
15976 let high_control = high_control + high_base;
15977 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
15978 let packed = crate::transmute::checked_transmute_copy(
15979 input[input_lane..].first_chunk::<16>().unwrap(),
15980 );
15981 let expanded = _mm_shuffle_epi8(packed, control);
15982 let result = expanded;
15983 crate::transmute::checked_transmute_store(
15984 result,
15985 output[16..].first_chunk_mut::<16>().unwrap(),
15986 );
15987 input_lane += block_bits.count_ones() as usize;
15988 let block_bits = _mm_movemask_epi8(_mm256_castsi256_si128(mask.val.0[1])) as usize;
15989 let low_mask = (block_bits) & 0xff;
15990 let high_mask = (block_bits) >> 8;
15991 let low_count = low_mask.count_ones() as u64;
15992 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
15993 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
15994 let high_base = low_count * 0x0101_0101_0101_0101;
15995 let high_control = high_control + high_base;
15996 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
15997 let packed = crate::transmute::checked_transmute_copy(
15998 input[input_lane..].first_chunk::<16>().unwrap(),
15999 );
16000 let expanded = _mm_shuffle_epi8(packed, control);
16001 let result = expanded;
16002 crate::transmute::checked_transmute_store(
16003 result,
16004 output[32..].first_chunk_mut::<16>().unwrap(),
16005 );
16006 input_lane += block_bits.count_ones() as usize;
16007 let block_bits =
16008 _mm_movemask_epi8(_mm256_extracti128_si256::<1>(mask.val.0[1])) as usize;
16009 let low_mask = (block_bits) & 0xff;
16010 let high_mask = (block_bits) >> 8;
16011 let low_count = low_mask.count_ones() as u64;
16012 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
16013 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
16014 let high_base = low_count * 0x0101_0101_0101_0101;
16015 let high_control = high_control + high_base;
16016 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
16017 let packed = crate::transmute::checked_transmute_copy(
16018 input[input_lane..].first_chunk::<16>().unwrap(),
16019 );
16020 let expanded = _mm_shuffle_epi8(packed, control);
16021 let result = expanded;
16022 crate::transmute::checked_transmute_store(
16023 result,
16024 output[48..].first_chunk_mut::<16>().unwrap(),
16025 );
16026 u8x64::simd_from(token, output)
16027 }
16028 );
16029 kernel(self, values, mask)
16030 }
16031 #[inline(always)]
16032 fn expand_merge_u8x64(
16033 self,
16034 values: u8x64<Self>,
16035 mask: mask8x64<Self>,
16036 merge: u8x64<Self>,
16037 ) -> u8x64<Self> {
16038 crate::kernel!(
16039 #[inline(always)]
16040 fn kernel(
16041 token: Avx2,
16042 values: u8x64<Avx2>,
16043 mask: mask8x64<Avx2>,
16044 merge: u8x64<Avx2>,
16045 ) -> u8x64<Avx2> {
16046 let input = <[u8; 64]>::from(values);
16047 let mut output = [0u8; 64];
16048 let mut input_lane = 0;
16049 let block_bits = _mm_movemask_epi8(_mm256_castsi256_si128(mask.val.0[0])) as usize;
16050 let low_mask = (block_bits) & 0xff;
16051 let high_mask = (block_bits) >> 8;
16052 let low_count = low_mask.count_ones() as u64;
16053 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
16054 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
16055 let high_base = low_count * 0x0101_0101_0101_0101;
16056 let high_control = high_control + high_base;
16057 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
16058 let packed = crate::transmute::checked_transmute_copy(
16059 input[input_lane..].first_chunk::<16>().unwrap(),
16060 );
16061 let expanded = _mm_shuffle_epi8(packed, control);
16062 let result = _mm_blendv_epi8(
16063 _mm256_castsi256_si128(merge.val.0[0]),
16064 expanded,
16065 _mm256_castsi256_si128(mask.val.0[0]),
16066 );
16067 crate::transmute::checked_transmute_store(
16068 result,
16069 output[0..].first_chunk_mut::<16>().unwrap(),
16070 );
16071 input_lane += block_bits.count_ones() as usize;
16072 let block_bits =
16073 _mm_movemask_epi8(_mm256_extracti128_si256::<1>(mask.val.0[0])) as usize;
16074 let low_mask = (block_bits) & 0xff;
16075 let high_mask = (block_bits) >> 8;
16076 let low_count = low_mask.count_ones() as u64;
16077 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
16078 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
16079 let high_base = low_count * 0x0101_0101_0101_0101;
16080 let high_control = high_control + high_base;
16081 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
16082 let packed = crate::transmute::checked_transmute_copy(
16083 input[input_lane..].first_chunk::<16>().unwrap(),
16084 );
16085 let expanded = _mm_shuffle_epi8(packed, control);
16086 let result = _mm_blendv_epi8(
16087 _mm256_extracti128_si256::<1>(merge.val.0[0]),
16088 expanded,
16089 _mm256_extracti128_si256::<1>(mask.val.0[0]),
16090 );
16091 crate::transmute::checked_transmute_store(
16092 result,
16093 output[16..].first_chunk_mut::<16>().unwrap(),
16094 );
16095 input_lane += block_bits.count_ones() as usize;
16096 let block_bits = _mm_movemask_epi8(_mm256_castsi256_si128(mask.val.0[1])) as usize;
16097 let low_mask = (block_bits) & 0xff;
16098 let high_mask = (block_bits) >> 8;
16099 let low_count = low_mask.count_ones() as u64;
16100 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
16101 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
16102 let high_base = low_count * 0x0101_0101_0101_0101;
16103 let high_control = high_control + high_base;
16104 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
16105 let packed = crate::transmute::checked_transmute_copy(
16106 input[input_lane..].first_chunk::<16>().unwrap(),
16107 );
16108 let expanded = _mm_shuffle_epi8(packed, control);
16109 let result = _mm_blendv_epi8(
16110 _mm256_castsi256_si128(merge.val.0[1]),
16111 expanded,
16112 _mm256_castsi256_si128(mask.val.0[1]),
16113 );
16114 crate::transmute::checked_transmute_store(
16115 result,
16116 output[32..].first_chunk_mut::<16>().unwrap(),
16117 );
16118 input_lane += block_bits.count_ones() as usize;
16119 let block_bits =
16120 _mm_movemask_epi8(_mm256_extracti128_si256::<1>(mask.val.0[1])) as usize;
16121 let low_mask = (block_bits) & 0xff;
16122 let high_mask = (block_bits) >> 8;
16123 let low_count = low_mask.count_ones() as u64;
16124 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
16125 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
16126 let high_base = low_count * 0x0101_0101_0101_0101;
16127 let high_control = high_control + high_base;
16128 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
16129 let packed = crate::transmute::checked_transmute_copy(
16130 input[input_lane..].first_chunk::<16>().unwrap(),
16131 );
16132 let expanded = _mm_shuffle_epi8(packed, control);
16133 let result = _mm_blendv_epi8(
16134 _mm256_extracti128_si256::<1>(merge.val.0[1]),
16135 expanded,
16136 _mm256_extracti128_si256::<1>(mask.val.0[1]),
16137 );
16138 crate::transmute::checked_transmute_store(
16139 result,
16140 output[48..].first_chunk_mut::<16>().unwrap(),
16141 );
16142 u8x64::simd_from(token, output)
16143 }
16144 );
16145 kernel(self, values, mask, merge)
16146 }
16147 #[inline(always)]
16148 fn split_u8x64(self, a: u8x64<Self>) -> (u8x32<Self>, u8x32<Self>) {
16149 (
16150 u8x32 {
16151 val: crate::support::Aligned256(a.val.0[0]),
16152 simd: self,
16153 },
16154 u8x32 {
16155 val: crate::support::Aligned256(a.val.0[1]),
16156 simd: self,
16157 },
16158 )
16159 }
16160 #[inline(always)]
16161 fn from_bitmask_mask8x64(self, bits: u64) -> mask8x64<Self> {
16162 crate::kernel!(
16163 #[inline(always)]
16164 fn kernel(token: Avx2, bits: u64) -> mask8x64<Avx2> {
16165 {
16166 let bit_bytes = _mm256_set1_epi64x(bits.cast_signed());
16167 let bit_mask = _mm256_setr_epi8(
16168 1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16,
16169 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128,
16170 );
16171 mask8x64 {
16172 val: crate::support::Aligned512([
16173 {
16174 let bit_bytes = _mm256_shuffle_epi8(
16175 bit_bytes,
16176 _mm256_setr_epi8(
16177 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2,
16178 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3,
16179 ),
16180 );
16181 _mm256_cmpeq_epi8(_mm256_and_si256(bit_bytes, bit_mask), bit_mask)
16182 },
16183 {
16184 let bit_bytes = _mm256_shuffle_epi8(
16185 bit_bytes,
16186 _mm256_setr_epi8(
16187 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6,
16188 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7,
16189 ),
16190 );
16191 _mm256_cmpeq_epi8(_mm256_and_si256(bit_bytes, bit_mask), bit_mask)
16192 },
16193 ]),
16194 simd: token,
16195 }
16196 }
16197 }
16198 );
16199 kernel(self, bits)
16200 }
16201 #[inline(always)]
16202 fn from_vector_mask8x64(self, vector: i8x64<Self>) -> mask8x64<Self> {
16203 mask8x64 {
16204 val: vector.val,
16205 simd: self,
16206 }
16207 }
16208 #[inline(always)]
16209 fn to_vector_mask8x64(self, a: mask8x64<Self>) -> i8x64<Self> {
16210 i8x64 {
16211 val: a.val,
16212 simd: self,
16213 }
16214 }
16215 #[inline(always)]
16216 fn set_mask8x64(self, a: &mut mask8x64<Self>, index: usize, value: bool) -> () {
16217 assert!(
16218 index < 64usize,
16219 "mask lane index {index} is out of bounds for {} lanes",
16220 64usize
16221 );
16222 let mut lanes: [i8; 64usize] = (*a).into();
16223 lanes[index] = if value { !0 } else { 0 };
16224 *a = lanes.simd_into(self);
16225 }
16226 #[inline(always)]
16227 fn rotate_elements_left_mask8x64<const OFFSET: usize>(
16228 self,
16229 a: mask8x64<Self>,
16230 ) -> mask8x64<Self> {
16231 let int = i8x64 {
16232 val: crate::transmute::checked_transmute_copy(&a.val),
16233 simd: self,
16234 };
16235 let rotated = int.rotate_elements_left::<OFFSET>();
16236 mask8x64 {
16237 val: crate::transmute::checked_transmute_copy(&rotated.val),
16238 simd: self,
16239 }
16240 }
16241 #[inline(always)]
16242 fn rotate_elements_right_mask8x64<const OFFSET: usize>(
16243 self,
16244 a: mask8x64<Self>,
16245 ) -> mask8x64<Self> {
16246 let int = i8x64 {
16247 val: crate::transmute::checked_transmute_copy(&a.val),
16248 simd: self,
16249 };
16250 let rotated = int.rotate_elements_right::<OFFSET>();
16251 mask8x64 {
16252 val: crate::transmute::checked_transmute_copy(&rotated.val),
16253 simd: self,
16254 }
16255 }
16256 #[inline(always)]
16257 fn split_mask8x64(self, a: mask8x64<Self>) -> (mask8x32<Self>, mask8x32<Self>) {
16258 (
16259 mask8x32 {
16260 val: crate::support::Aligned256(a.val.0[0]),
16261 simd: self,
16262 },
16263 mask8x32 {
16264 val: crate::support::Aligned256(a.val.0[1]),
16265 simd: self,
16266 },
16267 )
16268 }
16269 #[inline(always)]
16270 fn slide_i16x32<const SHIFT: usize>(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
16271 if SHIFT >= 32usize {
16272 return b;
16273 }
16274 let result = cross_block_alignr_256x2(
16275 self,
16276 Bytes::to_bytes(b).val.0,
16277 Bytes::to_bytes(a).val.0,
16278 SHIFT * 2usize,
16279 );
16280 Bytes::from_bytes(u8x64 {
16281 val: crate::support::Aligned512(result),
16282 simd: self,
16283 })
16284 }
16285 #[inline(always)]
16286 fn compress_i16x32(self, values: i16x32<Self>, mask: mask16x32<Self>) -> i16x32<Self> {
16287 let mask = mask8x64 {
16288 val: crate::transmute::checked_transmute_copy(&mask.val),
16289 simd: self,
16290 };
16291 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
16292 }
16293 #[inline(always)]
16294 fn compress_merge_i16x32(
16295 self,
16296 values: i16x32<Self>,
16297 mask: mask16x32<Self>,
16298 merge: i16x32<Self>,
16299 ) -> i16x32<Self> {
16300 let mask = mask8x64 {
16301 val: crate::transmute::checked_transmute_copy(&mask.val),
16302 simd: self,
16303 };
16304 Bytes::from_bytes(self.compress_merge_u8x64(
16305 Bytes::to_bytes(values),
16306 mask,
16307 Bytes::to_bytes(merge),
16308 ))
16309 }
16310 #[inline(always)]
16311 fn expand_i16x32(self, values: i16x32<Self>, mask: mask16x32<Self>) -> i16x32<Self> {
16312 let mask = mask8x64 {
16313 val: crate::transmute::checked_transmute_copy(&mask.val),
16314 simd: self,
16315 };
16316 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
16317 }
16318 #[inline(always)]
16319 fn expand_merge_i16x32(
16320 self,
16321 values: i16x32<Self>,
16322 mask: mask16x32<Self>,
16323 merge: i16x32<Self>,
16324 ) -> i16x32<Self> {
16325 let mask = mask8x64 {
16326 val: crate::transmute::checked_transmute_copy(&mask.val),
16327 simd: self,
16328 };
16329 Bytes::from_bytes(self.expand_merge_u8x64(
16330 Bytes::to_bytes(values),
16331 mask,
16332 Bytes::to_bytes(merge),
16333 ))
16334 }
16335 #[inline(always)]
16336 fn split_i16x32(self, a: i16x32<Self>) -> (i16x16<Self>, i16x16<Self>) {
16337 (
16338 i16x16 {
16339 val: crate::support::Aligned256(a.val.0[0]),
16340 simd: self,
16341 },
16342 i16x16 {
16343 val: crate::support::Aligned256(a.val.0[1]),
16344 simd: self,
16345 },
16346 )
16347 }
16348 #[inline(always)]
16349 fn slide_u16x32<const SHIFT: usize>(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
16350 if SHIFT >= 32usize {
16351 return b;
16352 }
16353 let result = cross_block_alignr_256x2(
16354 self,
16355 Bytes::to_bytes(b).val.0,
16356 Bytes::to_bytes(a).val.0,
16357 SHIFT * 2usize,
16358 );
16359 Bytes::from_bytes(u8x64 {
16360 val: crate::support::Aligned512(result),
16361 simd: self,
16362 })
16363 }
16364 #[inline(always)]
16365 fn compress_u16x32(self, values: u16x32<Self>, mask: mask16x32<Self>) -> u16x32<Self> {
16366 let mask = mask8x64 {
16367 val: crate::transmute::checked_transmute_copy(&mask.val),
16368 simd: self,
16369 };
16370 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
16371 }
16372 #[inline(always)]
16373 fn compress_merge_u16x32(
16374 self,
16375 values: u16x32<Self>,
16376 mask: mask16x32<Self>,
16377 merge: u16x32<Self>,
16378 ) -> u16x32<Self> {
16379 let mask = mask8x64 {
16380 val: crate::transmute::checked_transmute_copy(&mask.val),
16381 simd: self,
16382 };
16383 Bytes::from_bytes(self.compress_merge_u8x64(
16384 Bytes::to_bytes(values),
16385 mask,
16386 Bytes::to_bytes(merge),
16387 ))
16388 }
16389 #[inline(always)]
16390 fn expand_u16x32(self, values: u16x32<Self>, mask: mask16x32<Self>) -> u16x32<Self> {
16391 let mask = mask8x64 {
16392 val: crate::transmute::checked_transmute_copy(&mask.val),
16393 simd: self,
16394 };
16395 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
16396 }
16397 #[inline(always)]
16398 fn expand_merge_u16x32(
16399 self,
16400 values: u16x32<Self>,
16401 mask: mask16x32<Self>,
16402 merge: u16x32<Self>,
16403 ) -> u16x32<Self> {
16404 let mask = mask8x64 {
16405 val: crate::transmute::checked_transmute_copy(&mask.val),
16406 simd: self,
16407 };
16408 Bytes::from_bytes(self.expand_merge_u8x64(
16409 Bytes::to_bytes(values),
16410 mask,
16411 Bytes::to_bytes(merge),
16412 ))
16413 }
16414 #[inline(always)]
16415 fn split_u16x32(self, a: u16x32<Self>) -> (u16x16<Self>, u16x16<Self>) {
16416 (
16417 u16x16 {
16418 val: crate::support::Aligned256(a.val.0[0]),
16419 simd: self,
16420 },
16421 u16x16 {
16422 val: crate::support::Aligned256(a.val.0[1]),
16423 simd: self,
16424 },
16425 )
16426 }
16427 #[inline(always)]
16428 fn to_bitmask_mask16x32(self, a: mask16x32<Self>) -> u64 {
16429 crate::kernel!(
16430 #[inline(always)]
16431 fn kernel(token: Avx2, a: mask16x32<Avx2>) -> u64 {
16432 {
16433 let lo = _mm256_movemask_epi8(a.val.0[0]) as u32;
16434 let hi = _mm256_movemask_epi8(a.val.0[1]) as u32;
16435 let lo = _pext_u32(lo, 0x5555_5555u32) as u64;
16436 let hi = _pext_u32(hi, 0x5555_5555u32) as u64;
16437 lo | (hi << 16usize)
16438 }
16439 }
16440 );
16441 kernel(self, a)
16442 }
16443 #[inline(always)]
16444 fn from_vector_mask16x32(self, vector: i16x32<Self>) -> mask16x32<Self> {
16445 mask16x32 {
16446 val: vector.val,
16447 simd: self,
16448 }
16449 }
16450 #[inline(always)]
16451 fn to_vector_mask16x32(self, a: mask16x32<Self>) -> i16x32<Self> {
16452 i16x32 {
16453 val: a.val,
16454 simd: self,
16455 }
16456 }
16457 #[inline(always)]
16458 fn set_mask16x32(self, a: &mut mask16x32<Self>, index: usize, value: bool) -> () {
16459 assert!(
16460 index < 32usize,
16461 "mask lane index {index} is out of bounds for {} lanes",
16462 32usize
16463 );
16464 let mut lanes: [i16; 32usize] = (*a).into();
16465 lanes[index] = if value { !0 } else { 0 };
16466 *a = lanes.simd_into(self);
16467 }
16468 #[inline(always)]
16469 fn rotate_elements_left_mask16x32<const OFFSET: usize>(
16470 self,
16471 a: mask16x32<Self>,
16472 ) -> mask16x32<Self> {
16473 let int = i16x32 {
16474 val: crate::transmute::checked_transmute_copy(&a.val),
16475 simd: self,
16476 };
16477 let rotated = int.rotate_elements_left::<OFFSET>();
16478 mask16x32 {
16479 val: crate::transmute::checked_transmute_copy(&rotated.val),
16480 simd: self,
16481 }
16482 }
16483 #[inline(always)]
16484 fn rotate_elements_right_mask16x32<const OFFSET: usize>(
16485 self,
16486 a: mask16x32<Self>,
16487 ) -> mask16x32<Self> {
16488 let int = i16x32 {
16489 val: crate::transmute::checked_transmute_copy(&a.val),
16490 simd: self,
16491 };
16492 let rotated = int.rotate_elements_right::<OFFSET>();
16493 mask16x32 {
16494 val: crate::transmute::checked_transmute_copy(&rotated.val),
16495 simd: self,
16496 }
16497 }
16498 #[inline(always)]
16499 fn split_mask16x32(self, a: mask16x32<Self>) -> (mask16x16<Self>, mask16x16<Self>) {
16500 (
16501 mask16x16 {
16502 val: crate::support::Aligned256(a.val.0[0]),
16503 simd: self,
16504 },
16505 mask16x16 {
16506 val: crate::support::Aligned256(a.val.0[1]),
16507 simd: self,
16508 },
16509 )
16510 }
16511 #[inline(always)]
16512 fn slide_i32x16<const SHIFT: usize>(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
16513 if SHIFT >= 16usize {
16514 return b;
16515 }
16516 let result = cross_block_alignr_256x2(
16517 self,
16518 Bytes::to_bytes(b).val.0,
16519 Bytes::to_bytes(a).val.0,
16520 SHIFT * 4usize,
16521 );
16522 Bytes::from_bytes(u8x64 {
16523 val: crate::support::Aligned512(result),
16524 simd: self,
16525 })
16526 }
16527 #[inline(always)]
16528 fn compress_i32x16(self, values: i32x16<Self>, mask: mask32x16<Self>) -> i32x16<Self> {
16529 crate::kernel!(
16530 #[inline(always)]
16531 fn kernel(token: Avx2, values: i32x16<Avx2>, mask: mask32x16<Avx2>) -> i32x16<Avx2> {
16532 let (low_mask, high_mask) = token.split_mask32x16(mask);
16533 let low_count = low_mask.to_bitmask().count_ones() as usize;
16534 let (low, high) = values.split();
16535 let bits = mask.to_bitmask();
16536 if bits == 0 {
16537 return i32x16::splat(token, 0);
16538 }
16539 if bits == 65535 {
16540 return values;
16541 }
16542 let low = low.compress(low_mask);
16543 let high = high.compress(high_mask);
16544 let control: __m256i = crate::transmute::checked_transmute_copy(
16545 &crate::support::compact_256::SHIFTS[low_count],
16546 );
16547 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
16548 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
16549 let shifted = _mm256_permutevar8x32_epi32(high_bits, control);
16550 let result_low = _mm256_blendv_epi8(shifted, low_bits, control);
16551 let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control));
16552 i32x16 {
16553 val: crate::transmute::checked_transmute_copy(&[result_low, result_high]),
16554 simd: token,
16555 }
16556 }
16557 );
16558 kernel(self, values, mask)
16559 }
16560 #[inline(always)]
16561 fn compress_merge_i32x16(
16562 self,
16563 values: i32x16<Self>,
16564 mask: mask32x16<Self>,
16565 merge: i32x16<Self>,
16566 ) -> i32x16<Self> {
16567 crate::kernel!(
16568 #[inline(always)]
16569 fn kernel(
16570 token: Avx2,
16571 values: i32x16<Avx2>,
16572 mask: mask32x16<Avx2>,
16573 merge: i32x16<Avx2>,
16574 ) -> i32x16<Avx2> {
16575 let (low_mask, high_mask) = token.split_mask32x16(mask);
16576 let low_count = low_mask.to_bitmask().count_ones() as usize;
16577 let (low, high) = values.split();
16578 let bits = mask.to_bitmask();
16579 if bits == 0 {
16580 return merge;
16581 }
16582 if bits == 65535 {
16583 return values;
16584 }
16585 let low = low.compress(low_mask);
16586 let high = high.compress(high_mask);
16587 let control: __m256i = crate::transmute::checked_transmute_copy(
16588 &crate::support::compact_256::SHIFTS[low_count],
16589 );
16590 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
16591 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
16592 let shifted = _mm256_permutevar8x32_epi32(high_bits, control);
16593 let result_low = _mm256_blendv_epi8(shifted, low_bits, control);
16594 let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control));
16595 let compressed = i32x16 {
16596 val: crate::transmute::checked_transmute_copy(&[result_low, result_high]),
16597 simd: token,
16598 };
16599 let count = low_count + high_mask.to_bitmask().count_ones() as usize;
16600 let prefix = mask32x16::from_bitmask(token, (1u64 << count) - 1);
16601 token.select_i32x16(prefix, compressed, merge)
16602 }
16603 );
16604 kernel(self, values, mask, merge)
16605 }
16606 #[inline(always)]
16607 fn expand_i32x16(self, values: i32x16<Self>, mask: mask32x16<Self>) -> i32x16<Self> {
16608 crate::kernel!(
16609 #[inline(always)]
16610 fn kernel(token: Avx2, values: i32x16<Avx2>, mask: mask32x16<Avx2>) -> i32x16<Avx2> {
16611 let (low_mask, high_mask) = token.split_mask32x16(mask);
16612 let low_count = low_mask.to_bitmask().count_ones() as usize;
16613 let (low, high) = values.split();
16614 let control: __m256i = crate::transmute::checked_transmute_copy(
16615 &crate::support::compact_256::SHIFTS[8 - (low_count)],
16616 );
16617 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
16618 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
16619 let from_low = _mm256_permutevar8x32_epi32(low_bits, control);
16620 let from_high = _mm256_permutevar8x32_epi32(high_bits, control);
16621 let packed = _mm256_blendv_epi8(from_high, from_low, control);
16622 let high = i32x8 {
16623 val: crate::transmute::checked_transmute_copy(&packed),
16624 simd: token,
16625 };
16626 low.expand(low_mask).combine(high.expand(high_mask))
16627 }
16628 );
16629 kernel(self, values, mask)
16630 }
16631 #[inline(always)]
16632 fn expand_merge_i32x16(
16633 self,
16634 values: i32x16<Self>,
16635 mask: mask32x16<Self>,
16636 merge: i32x16<Self>,
16637 ) -> i32x16<Self> {
16638 crate::kernel!(
16639 #[inline(always)]
16640 fn kernel(
16641 token: Avx2,
16642 values: i32x16<Avx2>,
16643 mask: mask32x16<Avx2>,
16644 merge: i32x16<Avx2>,
16645 ) -> i32x16<Avx2> {
16646 let (low_mask, high_mask) = token.split_mask32x16(mask);
16647 let low_count = low_mask.to_bitmask().count_ones() as usize;
16648 let (low, high) = values.split();
16649 let control: __m256i = crate::transmute::checked_transmute_copy(
16650 &crate::support::compact_256::SHIFTS[8 - (low_count)],
16651 );
16652 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
16653 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
16654 let from_low = _mm256_permutevar8x32_epi32(low_bits, control);
16655 let from_high = _mm256_permutevar8x32_epi32(high_bits, control);
16656 let packed = _mm256_blendv_epi8(from_high, from_low, control);
16657 let high = i32x8 {
16658 val: crate::transmute::checked_transmute_copy(&packed),
16659 simd: token,
16660 };
16661 let (low_merge, high_merge) = merge.split();
16662 low.expand_merge(low_mask, low_merge)
16663 .combine(high.expand_merge(high_mask, high_merge))
16664 }
16665 );
16666 kernel(self, values, mask, merge)
16667 }
16668 #[inline(always)]
16669 fn split_i32x16(self, a: i32x16<Self>) -> (i32x8<Self>, i32x8<Self>) {
16670 (
16671 i32x8 {
16672 val: crate::support::Aligned256(a.val.0[0]),
16673 simd: self,
16674 },
16675 i32x8 {
16676 val: crate::support::Aligned256(a.val.0[1]),
16677 simd: self,
16678 },
16679 )
16680 }
16681 #[inline(always)]
16682 fn slide_u32x16<const SHIFT: usize>(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
16683 if SHIFT >= 16usize {
16684 return b;
16685 }
16686 let result = cross_block_alignr_256x2(
16687 self,
16688 Bytes::to_bytes(b).val.0,
16689 Bytes::to_bytes(a).val.0,
16690 SHIFT * 4usize,
16691 );
16692 Bytes::from_bytes(u8x64 {
16693 val: crate::support::Aligned512(result),
16694 simd: self,
16695 })
16696 }
16697 #[inline(always)]
16698 fn compress_u32x16(self, values: u32x16<Self>, mask: mask32x16<Self>) -> u32x16<Self> {
16699 crate::kernel!(
16700 #[inline(always)]
16701 fn kernel(token: Avx2, values: u32x16<Avx2>, mask: mask32x16<Avx2>) -> u32x16<Avx2> {
16702 let (low_mask, high_mask) = token.split_mask32x16(mask);
16703 let low_count = low_mask.to_bitmask().count_ones() as usize;
16704 let (low, high) = values.split();
16705 let bits = mask.to_bitmask();
16706 if bits == 0 {
16707 return u32x16::splat(token, 0);
16708 }
16709 if bits == 65535 {
16710 return values;
16711 }
16712 let low = low.compress(low_mask);
16713 let high = high.compress(high_mask);
16714 let control: __m256i = crate::transmute::checked_transmute_copy(
16715 &crate::support::compact_256::SHIFTS[low_count],
16716 );
16717 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
16718 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
16719 let shifted = _mm256_permutevar8x32_epi32(high_bits, control);
16720 let result_low = _mm256_blendv_epi8(shifted, low_bits, control);
16721 let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control));
16722 u32x16 {
16723 val: crate::transmute::checked_transmute_copy(&[result_low, result_high]),
16724 simd: token,
16725 }
16726 }
16727 );
16728 kernel(self, values, mask)
16729 }
16730 #[inline(always)]
16731 fn compress_merge_u32x16(
16732 self,
16733 values: u32x16<Self>,
16734 mask: mask32x16<Self>,
16735 merge: u32x16<Self>,
16736 ) -> u32x16<Self> {
16737 crate::kernel!(
16738 #[inline(always)]
16739 fn kernel(
16740 token: Avx2,
16741 values: u32x16<Avx2>,
16742 mask: mask32x16<Avx2>,
16743 merge: u32x16<Avx2>,
16744 ) -> u32x16<Avx2> {
16745 let (low_mask, high_mask) = token.split_mask32x16(mask);
16746 let low_count = low_mask.to_bitmask().count_ones() as usize;
16747 let (low, high) = values.split();
16748 let bits = mask.to_bitmask();
16749 if bits == 0 {
16750 return merge;
16751 }
16752 if bits == 65535 {
16753 return values;
16754 }
16755 let low = low.compress(low_mask);
16756 let high = high.compress(high_mask);
16757 let control: __m256i = crate::transmute::checked_transmute_copy(
16758 &crate::support::compact_256::SHIFTS[low_count],
16759 );
16760 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
16761 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
16762 let shifted = _mm256_permutevar8x32_epi32(high_bits, control);
16763 let result_low = _mm256_blendv_epi8(shifted, low_bits, control);
16764 let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control));
16765 let compressed = u32x16 {
16766 val: crate::transmute::checked_transmute_copy(&[result_low, result_high]),
16767 simd: token,
16768 };
16769 let count = low_count + high_mask.to_bitmask().count_ones() as usize;
16770 let prefix = mask32x16::from_bitmask(token, (1u64 << count) - 1);
16771 token.select_u32x16(prefix, compressed, merge)
16772 }
16773 );
16774 kernel(self, values, mask, merge)
16775 }
16776 #[inline(always)]
16777 fn expand_u32x16(self, values: u32x16<Self>, mask: mask32x16<Self>) -> u32x16<Self> {
16778 crate::kernel!(
16779 #[inline(always)]
16780 fn kernel(token: Avx2, values: u32x16<Avx2>, mask: mask32x16<Avx2>) -> u32x16<Avx2> {
16781 let (low_mask, high_mask) = token.split_mask32x16(mask);
16782 let low_count = low_mask.to_bitmask().count_ones() as usize;
16783 let (low, high) = values.split();
16784 let control: __m256i = crate::transmute::checked_transmute_copy(
16785 &crate::support::compact_256::SHIFTS[8 - (low_count)],
16786 );
16787 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
16788 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
16789 let from_low = _mm256_permutevar8x32_epi32(low_bits, control);
16790 let from_high = _mm256_permutevar8x32_epi32(high_bits, control);
16791 let packed = _mm256_blendv_epi8(from_high, from_low, control);
16792 let high = u32x8 {
16793 val: crate::transmute::checked_transmute_copy(&packed),
16794 simd: token,
16795 };
16796 low.expand(low_mask).combine(high.expand(high_mask))
16797 }
16798 );
16799 kernel(self, values, mask)
16800 }
16801 #[inline(always)]
16802 fn expand_merge_u32x16(
16803 self,
16804 values: u32x16<Self>,
16805 mask: mask32x16<Self>,
16806 merge: u32x16<Self>,
16807 ) -> u32x16<Self> {
16808 crate::kernel!(
16809 #[inline(always)]
16810 fn kernel(
16811 token: Avx2,
16812 values: u32x16<Avx2>,
16813 mask: mask32x16<Avx2>,
16814 merge: u32x16<Avx2>,
16815 ) -> u32x16<Avx2> {
16816 let (low_mask, high_mask) = token.split_mask32x16(mask);
16817 let low_count = low_mask.to_bitmask().count_ones() as usize;
16818 let (low, high) = values.split();
16819 let control: __m256i = crate::transmute::checked_transmute_copy(
16820 &crate::support::compact_256::SHIFTS[8 - (low_count)],
16821 );
16822 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
16823 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
16824 let from_low = _mm256_permutevar8x32_epi32(low_bits, control);
16825 let from_high = _mm256_permutevar8x32_epi32(high_bits, control);
16826 let packed = _mm256_blendv_epi8(from_high, from_low, control);
16827 let high = u32x8 {
16828 val: crate::transmute::checked_transmute_copy(&packed),
16829 simd: token,
16830 };
16831 let (low_merge, high_merge) = merge.split();
16832 low.expand_merge(low_mask, low_merge)
16833 .combine(high.expand_merge(high_mask, high_merge))
16834 }
16835 );
16836 kernel(self, values, mask, merge)
16837 }
16838 #[inline(always)]
16839 fn split_u32x16(self, a: u32x16<Self>) -> (u32x8<Self>, u32x8<Self>) {
16840 (
16841 u32x8 {
16842 val: crate::support::Aligned256(a.val.0[0]),
16843 simd: self,
16844 },
16845 u32x8 {
16846 val: crate::support::Aligned256(a.val.0[1]),
16847 simd: self,
16848 },
16849 )
16850 }
16851 #[inline(always)]
16852 fn from_bitmask_mask32x16(self, bits: u64) -> mask32x16<Self> {
16853 crate::kernel!(
16854 #[inline(always)]
16855 fn kernel(token: Avx2, bits: u64) -> mask32x16<Avx2> {
16856 {
16857 let bit_lanes = _mm256_set1_epi32(bits as i32);
16858 mask32x16 {
16859 val: crate::support::Aligned512([
16860 {
16861 let bit_mask = _mm256_setr_epi32(1, 2, 4, 8, 16, 32, 64, 128);
16862 _mm256_cmpeq_epi32(_mm256_and_si256(bit_lanes, bit_mask), bit_mask)
16863 },
16864 {
16865 let bit_mask = _mm256_setr_epi32(
16866 256, 512, 1024, 2048, 4096, 8192, 16384, 32768,
16867 );
16868 _mm256_cmpeq_epi32(_mm256_and_si256(bit_lanes, bit_mask), bit_mask)
16869 },
16870 ]),
16871 simd: token,
16872 }
16873 }
16874 }
16875 );
16876 kernel(self, bits)
16877 }
16878 #[inline(always)]
16879 fn from_vector_mask32x16(self, vector: i32x16<Self>) -> mask32x16<Self> {
16880 mask32x16 {
16881 val: vector.val,
16882 simd: self,
16883 }
16884 }
16885 #[inline(always)]
16886 fn to_vector_mask32x16(self, a: mask32x16<Self>) -> i32x16<Self> {
16887 i32x16 {
16888 val: a.val,
16889 simd: self,
16890 }
16891 }
16892 #[inline(always)]
16893 fn set_mask32x16(self, a: &mut mask32x16<Self>, index: usize, value: bool) -> () {
16894 assert!(
16895 index < 16usize,
16896 "mask lane index {index} is out of bounds for {} lanes",
16897 16usize
16898 );
16899 let mut lanes: [i32; 16usize] = (*a).into();
16900 lanes[index] = if value { !0 } else { 0 };
16901 *a = lanes.simd_into(self);
16902 }
16903 #[inline(always)]
16904 fn rotate_elements_left_mask32x16<const OFFSET: usize>(
16905 self,
16906 a: mask32x16<Self>,
16907 ) -> mask32x16<Self> {
16908 let int = i32x16 {
16909 val: crate::transmute::checked_transmute_copy(&a.val),
16910 simd: self,
16911 };
16912 let rotated = int.rotate_elements_left::<OFFSET>();
16913 mask32x16 {
16914 val: crate::transmute::checked_transmute_copy(&rotated.val),
16915 simd: self,
16916 }
16917 }
16918 #[inline(always)]
16919 fn rotate_elements_right_mask32x16<const OFFSET: usize>(
16920 self,
16921 a: mask32x16<Self>,
16922 ) -> mask32x16<Self> {
16923 let int = i32x16 {
16924 val: crate::transmute::checked_transmute_copy(&a.val),
16925 simd: self,
16926 };
16927 let rotated = int.rotate_elements_right::<OFFSET>();
16928 mask32x16 {
16929 val: crate::transmute::checked_transmute_copy(&rotated.val),
16930 simd: self,
16931 }
16932 }
16933 #[inline(always)]
16934 fn split_mask32x16(self, a: mask32x16<Self>) -> (mask32x8<Self>, mask32x8<Self>) {
16935 (
16936 mask32x8 {
16937 val: crate::support::Aligned256(a.val.0[0]),
16938 simd: self,
16939 },
16940 mask32x8 {
16941 val: crate::support::Aligned256(a.val.0[1]),
16942 simd: self,
16943 },
16944 )
16945 }
16946 #[inline(always)]
16947 fn slide_f64x8<const SHIFT: usize>(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
16948 if SHIFT >= 8usize {
16949 return b;
16950 }
16951 let result = cross_block_alignr_256x2(
16952 self,
16953 Bytes::to_bytes(b).val.0,
16954 Bytes::to_bytes(a).val.0,
16955 SHIFT * 8usize,
16956 );
16957 Bytes::from_bytes(u8x64 {
16958 val: crate::support::Aligned512(result),
16959 simd: self,
16960 })
16961 }
16962 #[inline(always)]
16963 fn compress_f64x8(self, values: f64x8<Self>, mask: mask64x8<Self>) -> f64x8<Self> {
16964 crate::kernel!(
16965 #[inline(always)]
16966 fn kernel(token: Avx2, values: f64x8<Avx2>, mask: mask64x8<Avx2>) -> f64x8<Avx2> {
16967 let (low_mask, high_mask) = token.split_mask64x8(mask);
16968 let low_count = low_mask.to_bitmask().count_ones() as usize;
16969 let (low, high) = values.split();
16970 let bits = mask.to_bitmask();
16971 if bits == 0 {
16972 return f64x8::splat(token, 0.0);
16973 }
16974 if bits == 255 {
16975 return values;
16976 }
16977 let low = low.compress(low_mask);
16978 let high = high.compress(high_mask);
16979 let control: __m256i = crate::transmute::checked_transmute_copy(
16980 &crate::support::compact_256::SHIFTS[low_count * 2],
16981 );
16982 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
16983 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
16984 let shifted = _mm256_permutevar8x32_epi32(high_bits, control);
16985 let result_low = _mm256_blendv_epi8(shifted, low_bits, control);
16986 let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control));
16987 f64x8 {
16988 val: crate::transmute::checked_transmute_copy(&[result_low, result_high]),
16989 simd: token,
16990 }
16991 }
16992 );
16993 kernel(self, values, mask)
16994 }
16995 #[inline(always)]
16996 fn compress_merge_f64x8(
16997 self,
16998 values: f64x8<Self>,
16999 mask: mask64x8<Self>,
17000 merge: f64x8<Self>,
17001 ) -> f64x8<Self> {
17002 crate::kernel!(
17003 #[inline(always)]
17004 fn kernel(
17005 token: Avx2,
17006 values: f64x8<Avx2>,
17007 mask: mask64x8<Avx2>,
17008 merge: f64x8<Avx2>,
17009 ) -> f64x8<Avx2> {
17010 let (low_mask, high_mask) = token.split_mask64x8(mask);
17011 let low_count = low_mask.to_bitmask().count_ones() as usize;
17012 let (low, high) = values.split();
17013 let bits = mask.to_bitmask();
17014 if bits == 0 {
17015 return merge;
17016 }
17017 if bits == 255 {
17018 return values;
17019 }
17020 let low = low.compress(low_mask);
17021 let high = high.compress(high_mask);
17022 let control: __m256i = crate::transmute::checked_transmute_copy(
17023 &crate::support::compact_256::SHIFTS[low_count * 2],
17024 );
17025 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
17026 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
17027 let shifted = _mm256_permutevar8x32_epi32(high_bits, control);
17028 let result_low = _mm256_blendv_epi8(shifted, low_bits, control);
17029 let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control));
17030 let compressed = f64x8 {
17031 val: crate::transmute::checked_transmute_copy(&[result_low, result_high]),
17032 simd: token,
17033 };
17034 let count = low_count + high_mask.to_bitmask().count_ones() as usize;
17035 let prefix = mask64x8::from_bitmask(token, (1u64 << count) - 1);
17036 token.select_f64x8(prefix, compressed, merge)
17037 }
17038 );
17039 kernel(self, values, mask, merge)
17040 }
17041 #[inline(always)]
17042 fn expand_f64x8(self, values: f64x8<Self>, mask: mask64x8<Self>) -> f64x8<Self> {
17043 crate::kernel!(
17044 #[inline(always)]
17045 fn kernel(token: Avx2, values: f64x8<Avx2>, mask: mask64x8<Avx2>) -> f64x8<Avx2> {
17046 let (low_mask, high_mask) = token.split_mask64x8(mask);
17047 let low_count = low_mask.to_bitmask().count_ones() as usize;
17048 let (low, high) = values.split();
17049 let control: __m256i = crate::transmute::checked_transmute_copy(
17050 &crate::support::compact_256::SHIFTS[8 - (low_count * 2)],
17051 );
17052 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
17053 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
17054 let from_low = _mm256_permutevar8x32_epi32(low_bits, control);
17055 let from_high = _mm256_permutevar8x32_epi32(high_bits, control);
17056 let packed = _mm256_blendv_epi8(from_high, from_low, control);
17057 let high = f64x4 {
17058 val: crate::transmute::checked_transmute_copy(&packed),
17059 simd: token,
17060 };
17061 low.expand(low_mask).combine(high.expand(high_mask))
17062 }
17063 );
17064 kernel(self, values, mask)
17065 }
17066 #[inline(always)]
17067 fn expand_merge_f64x8(
17068 self,
17069 values: f64x8<Self>,
17070 mask: mask64x8<Self>,
17071 merge: f64x8<Self>,
17072 ) -> f64x8<Self> {
17073 crate::kernel!(
17074 #[inline(always)]
17075 fn kernel(
17076 token: Avx2,
17077 values: f64x8<Avx2>,
17078 mask: mask64x8<Avx2>,
17079 merge: f64x8<Avx2>,
17080 ) -> f64x8<Avx2> {
17081 let (low_mask, high_mask) = token.split_mask64x8(mask);
17082 let low_count = low_mask.to_bitmask().count_ones() as usize;
17083 let (low, high) = values.split();
17084 let control: __m256i = crate::transmute::checked_transmute_copy(
17085 &crate::support::compact_256::SHIFTS[8 - (low_count * 2)],
17086 );
17087 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
17088 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
17089 let from_low = _mm256_permutevar8x32_epi32(low_bits, control);
17090 let from_high = _mm256_permutevar8x32_epi32(high_bits, control);
17091 let packed = _mm256_blendv_epi8(from_high, from_low, control);
17092 let high = f64x4 {
17093 val: crate::transmute::checked_transmute_copy(&packed),
17094 simd: token,
17095 };
17096 let (low_merge, high_merge) = merge.split();
17097 low.expand_merge(low_mask, low_merge)
17098 .combine(high.expand_merge(high_mask, high_merge))
17099 }
17100 );
17101 kernel(self, values, mask, merge)
17102 }
17103 #[inline(always)]
17104 fn simd_ne_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> mask64x8<Self> {
17105 let (a0, a1) = self.split_f64x8(a);
17106 let (b0, b1) = self.split_f64x8(b);
17107 self.combine_mask64x4(self.simd_ne_f64x4(a0, b0), self.simd_ne_f64x4(a1, b1))
17108 }
17109 #[inline(always)]
17110 fn is_nan_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
17111 a.simd_ne(a)
17112 }
17113 #[inline(always)]
17114 fn is_infinite_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
17115 a.abs().simd_eq(f64::INFINITY)
17116 }
17117 #[inline(always)]
17118 fn is_finite_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
17119 a.abs().simd_lt(f64::INFINITY)
17120 }
17121 #[inline(always)]
17122 fn is_subnormal_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
17123 let i: i64x8<Self> = a.bitcast();
17124 (i & 0x7FF0_0000_0000_0000_i64).simd_eq(0) & (i & 0x000F_FFFF_FFFF_FFFF_i64).simd_ne(0)
17125 }
17126 #[inline(always)]
17127 fn is_normal_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
17128 let i: i64x8<Self> = a.bitcast();
17129 let exp = i & 0x7FF0_0000_0000_0000_i64;
17130 exp.simd_ne(0) & exp.simd_ne(0x7FF0_0000_0000_0000_i64)
17131 }
17132 #[inline(always)]
17133 fn is_sign_positive_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
17134 let i: i64x8<Self> = a.bitcast();
17135 (i & i64::MIN).simd_eq(0)
17136 }
17137 #[inline(always)]
17138 fn is_sign_negative_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
17139 !a.is_sign_positive()
17140 }
17141 #[inline(always)]
17142 fn split_f64x8(self, a: f64x8<Self>) -> (f64x4<Self>, f64x4<Self>) {
17143 (
17144 f64x4 {
17145 val: crate::support::Aligned256(a.val.0[0]),
17146 simd: self,
17147 },
17148 f64x4 {
17149 val: crate::support::Aligned256(a.val.0[1]),
17150 simd: self,
17151 },
17152 )
17153 }
17154 #[inline(always)]
17155 fn slide_i64x8<const SHIFT: usize>(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
17156 if SHIFT >= 8usize {
17157 return b;
17158 }
17159 let result = cross_block_alignr_256x2(
17160 self,
17161 Bytes::to_bytes(b).val.0,
17162 Bytes::to_bytes(a).val.0,
17163 SHIFT * 8usize,
17164 );
17165 Bytes::from_bytes(u8x64 {
17166 val: crate::support::Aligned512(result),
17167 simd: self,
17168 })
17169 }
17170 #[inline(always)]
17171 fn compress_i64x8(self, values: i64x8<Self>, mask: mask64x8<Self>) -> i64x8<Self> {
17172 crate::kernel!(
17173 #[inline(always)]
17174 fn kernel(token: Avx2, values: i64x8<Avx2>, mask: mask64x8<Avx2>) -> i64x8<Avx2> {
17175 let (low_mask, high_mask) = token.split_mask64x8(mask);
17176 let low_count = low_mask.to_bitmask().count_ones() as usize;
17177 let (low, high) = values.split();
17178 let bits = mask.to_bitmask();
17179 if bits == 0 {
17180 return i64x8::splat(token, 0);
17181 }
17182 if bits == 255 {
17183 return values;
17184 }
17185 let low = low.compress(low_mask);
17186 let high = high.compress(high_mask);
17187 let control: __m256i = crate::transmute::checked_transmute_copy(
17188 &crate::support::compact_256::SHIFTS[low_count * 2],
17189 );
17190 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
17191 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
17192 let shifted = _mm256_permutevar8x32_epi32(high_bits, control);
17193 let result_low = _mm256_blendv_epi8(shifted, low_bits, control);
17194 let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control));
17195 i64x8 {
17196 val: crate::transmute::checked_transmute_copy(&[result_low, result_high]),
17197 simd: token,
17198 }
17199 }
17200 );
17201 kernel(self, values, mask)
17202 }
17203 #[inline(always)]
17204 fn compress_merge_i64x8(
17205 self,
17206 values: i64x8<Self>,
17207 mask: mask64x8<Self>,
17208 merge: i64x8<Self>,
17209 ) -> i64x8<Self> {
17210 crate::kernel!(
17211 #[inline(always)]
17212 fn kernel(
17213 token: Avx2,
17214 values: i64x8<Avx2>,
17215 mask: mask64x8<Avx2>,
17216 merge: i64x8<Avx2>,
17217 ) -> i64x8<Avx2> {
17218 let (low_mask, high_mask) = token.split_mask64x8(mask);
17219 let low_count = low_mask.to_bitmask().count_ones() as usize;
17220 let (low, high) = values.split();
17221 let bits = mask.to_bitmask();
17222 if bits == 0 {
17223 return merge;
17224 }
17225 if bits == 255 {
17226 return values;
17227 }
17228 let low = low.compress(low_mask);
17229 let high = high.compress(high_mask);
17230 let control: __m256i = crate::transmute::checked_transmute_copy(
17231 &crate::support::compact_256::SHIFTS[low_count * 2],
17232 );
17233 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
17234 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
17235 let shifted = _mm256_permutevar8x32_epi32(high_bits, control);
17236 let result_low = _mm256_blendv_epi8(shifted, low_bits, control);
17237 let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control));
17238 let compressed = i64x8 {
17239 val: crate::transmute::checked_transmute_copy(&[result_low, result_high]),
17240 simd: token,
17241 };
17242 let count = low_count + high_mask.to_bitmask().count_ones() as usize;
17243 let prefix = mask64x8::from_bitmask(token, (1u64 << count) - 1);
17244 token.select_i64x8(prefix, compressed, merge)
17245 }
17246 );
17247 kernel(self, values, mask, merge)
17248 }
17249 #[inline(always)]
17250 fn expand_i64x8(self, values: i64x8<Self>, mask: mask64x8<Self>) -> i64x8<Self> {
17251 crate::kernel!(
17252 #[inline(always)]
17253 fn kernel(token: Avx2, values: i64x8<Avx2>, mask: mask64x8<Avx2>) -> i64x8<Avx2> {
17254 let (low_mask, high_mask) = token.split_mask64x8(mask);
17255 let low_count = low_mask.to_bitmask().count_ones() as usize;
17256 let (low, high) = values.split();
17257 let control: __m256i = crate::transmute::checked_transmute_copy(
17258 &crate::support::compact_256::SHIFTS[8 - (low_count * 2)],
17259 );
17260 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
17261 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
17262 let from_low = _mm256_permutevar8x32_epi32(low_bits, control);
17263 let from_high = _mm256_permutevar8x32_epi32(high_bits, control);
17264 let packed = _mm256_blendv_epi8(from_high, from_low, control);
17265 let high = i64x4 {
17266 val: crate::transmute::checked_transmute_copy(&packed),
17267 simd: token,
17268 };
17269 low.expand(low_mask).combine(high.expand(high_mask))
17270 }
17271 );
17272 kernel(self, values, mask)
17273 }
17274 #[inline(always)]
17275 fn expand_merge_i64x8(
17276 self,
17277 values: i64x8<Self>,
17278 mask: mask64x8<Self>,
17279 merge: i64x8<Self>,
17280 ) -> i64x8<Self> {
17281 crate::kernel!(
17282 #[inline(always)]
17283 fn kernel(
17284 token: Avx2,
17285 values: i64x8<Avx2>,
17286 mask: mask64x8<Avx2>,
17287 merge: i64x8<Avx2>,
17288 ) -> i64x8<Avx2> {
17289 let (low_mask, high_mask) = token.split_mask64x8(mask);
17290 let low_count = low_mask.to_bitmask().count_ones() as usize;
17291 let (low, high) = values.split();
17292 let control: __m256i = crate::transmute::checked_transmute_copy(
17293 &crate::support::compact_256::SHIFTS[8 - (low_count * 2)],
17294 );
17295 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
17296 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
17297 let from_low = _mm256_permutevar8x32_epi32(low_bits, control);
17298 let from_high = _mm256_permutevar8x32_epi32(high_bits, control);
17299 let packed = _mm256_blendv_epi8(from_high, from_low, control);
17300 let high = i64x4 {
17301 val: crate::transmute::checked_transmute_copy(&packed),
17302 simd: token,
17303 };
17304 let (low_merge, high_merge) = merge.split();
17305 low.expand_merge(low_mask, low_merge)
17306 .combine(high.expand_merge(high_mask, high_merge))
17307 }
17308 );
17309 kernel(self, values, mask, merge)
17310 }
17311 #[inline(always)]
17312 fn split_i64x8(self, a: i64x8<Self>) -> (i64x4<Self>, i64x4<Self>) {
17313 (
17314 i64x4 {
17315 val: crate::support::Aligned256(a.val.0[0]),
17316 simd: self,
17317 },
17318 i64x4 {
17319 val: crate::support::Aligned256(a.val.0[1]),
17320 simd: self,
17321 },
17322 )
17323 }
17324 #[inline(always)]
17325 fn slide_u64x8<const SHIFT: usize>(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
17326 if SHIFT >= 8usize {
17327 return b;
17328 }
17329 let result = cross_block_alignr_256x2(
17330 self,
17331 Bytes::to_bytes(b).val.0,
17332 Bytes::to_bytes(a).val.0,
17333 SHIFT * 8usize,
17334 );
17335 Bytes::from_bytes(u8x64 {
17336 val: crate::support::Aligned512(result),
17337 simd: self,
17338 })
17339 }
17340 #[inline(always)]
17341 fn compress_u64x8(self, values: u64x8<Self>, mask: mask64x8<Self>) -> u64x8<Self> {
17342 crate::kernel!(
17343 #[inline(always)]
17344 fn kernel(token: Avx2, values: u64x8<Avx2>, mask: mask64x8<Avx2>) -> u64x8<Avx2> {
17345 let (low_mask, high_mask) = token.split_mask64x8(mask);
17346 let low_count = low_mask.to_bitmask().count_ones() as usize;
17347 let (low, high) = values.split();
17348 let bits = mask.to_bitmask();
17349 if bits == 0 {
17350 return u64x8::splat(token, 0);
17351 }
17352 if bits == 255 {
17353 return values;
17354 }
17355 let low = low.compress(low_mask);
17356 let high = high.compress(high_mask);
17357 let control: __m256i = crate::transmute::checked_transmute_copy(
17358 &crate::support::compact_256::SHIFTS[low_count * 2],
17359 );
17360 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
17361 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
17362 let shifted = _mm256_permutevar8x32_epi32(high_bits, control);
17363 let result_low = _mm256_blendv_epi8(shifted, low_bits, control);
17364 let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control));
17365 u64x8 {
17366 val: crate::transmute::checked_transmute_copy(&[result_low, result_high]),
17367 simd: token,
17368 }
17369 }
17370 );
17371 kernel(self, values, mask)
17372 }
17373 #[inline(always)]
17374 fn compress_merge_u64x8(
17375 self,
17376 values: u64x8<Self>,
17377 mask: mask64x8<Self>,
17378 merge: u64x8<Self>,
17379 ) -> u64x8<Self> {
17380 crate::kernel!(
17381 #[inline(always)]
17382 fn kernel(
17383 token: Avx2,
17384 values: u64x8<Avx2>,
17385 mask: mask64x8<Avx2>,
17386 merge: u64x8<Avx2>,
17387 ) -> u64x8<Avx2> {
17388 let (low_mask, high_mask) = token.split_mask64x8(mask);
17389 let low_count = low_mask.to_bitmask().count_ones() as usize;
17390 let (low, high) = values.split();
17391 let bits = mask.to_bitmask();
17392 if bits == 0 {
17393 return merge;
17394 }
17395 if bits == 255 {
17396 return values;
17397 }
17398 let low = low.compress(low_mask);
17399 let high = high.compress(high_mask);
17400 let control: __m256i = crate::transmute::checked_transmute_copy(
17401 &crate::support::compact_256::SHIFTS[low_count * 2],
17402 );
17403 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
17404 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
17405 let shifted = _mm256_permutevar8x32_epi32(high_bits, control);
17406 let result_low = _mm256_blendv_epi8(shifted, low_bits, control);
17407 let result_high = _mm256_and_si256(shifted, _mm256_srai_epi32::<31>(control));
17408 let compressed = u64x8 {
17409 val: crate::transmute::checked_transmute_copy(&[result_low, result_high]),
17410 simd: token,
17411 };
17412 let count = low_count + high_mask.to_bitmask().count_ones() as usize;
17413 let prefix = mask64x8::from_bitmask(token, (1u64 << count) - 1);
17414 token.select_u64x8(prefix, compressed, merge)
17415 }
17416 );
17417 kernel(self, values, mask, merge)
17418 }
17419 #[inline(always)]
17420 fn expand_u64x8(self, values: u64x8<Self>, mask: mask64x8<Self>) -> u64x8<Self> {
17421 crate::kernel!(
17422 #[inline(always)]
17423 fn kernel(token: Avx2, values: u64x8<Avx2>, mask: mask64x8<Avx2>) -> u64x8<Avx2> {
17424 let (low_mask, high_mask) = token.split_mask64x8(mask);
17425 let low_count = low_mask.to_bitmask().count_ones() as usize;
17426 let (low, high) = values.split();
17427 let control: __m256i = crate::transmute::checked_transmute_copy(
17428 &crate::support::compact_256::SHIFTS[8 - (low_count * 2)],
17429 );
17430 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
17431 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
17432 let from_low = _mm256_permutevar8x32_epi32(low_bits, control);
17433 let from_high = _mm256_permutevar8x32_epi32(high_bits, control);
17434 let packed = _mm256_blendv_epi8(from_high, from_low, control);
17435 let high = u64x4 {
17436 val: crate::transmute::checked_transmute_copy(&packed),
17437 simd: token,
17438 };
17439 low.expand(low_mask).combine(high.expand(high_mask))
17440 }
17441 );
17442 kernel(self, values, mask)
17443 }
17444 #[inline(always)]
17445 fn expand_merge_u64x8(
17446 self,
17447 values: u64x8<Self>,
17448 mask: mask64x8<Self>,
17449 merge: u64x8<Self>,
17450 ) -> u64x8<Self> {
17451 crate::kernel!(
17452 #[inline(always)]
17453 fn kernel(
17454 token: Avx2,
17455 values: u64x8<Avx2>,
17456 mask: mask64x8<Avx2>,
17457 merge: u64x8<Avx2>,
17458 ) -> u64x8<Avx2> {
17459 let (low_mask, high_mask) = token.split_mask64x8(mask);
17460 let low_count = low_mask.to_bitmask().count_ones() as usize;
17461 let (low, high) = values.split();
17462 let control: __m256i = crate::transmute::checked_transmute_copy(
17463 &crate::support::compact_256::SHIFTS[8 - (low_count * 2)],
17464 );
17465 let low_bits = crate::transmute::checked_transmute_copy(&low.val);
17466 let high_bits = crate::transmute::checked_transmute_copy(&high.val);
17467 let from_low = _mm256_permutevar8x32_epi32(low_bits, control);
17468 let from_high = _mm256_permutevar8x32_epi32(high_bits, control);
17469 let packed = _mm256_blendv_epi8(from_high, from_low, control);
17470 let high = u64x4 {
17471 val: crate::transmute::checked_transmute_copy(&packed),
17472 simd: token,
17473 };
17474 let (low_merge, high_merge) = merge.split();
17475 low.expand_merge(low_mask, low_merge)
17476 .combine(high.expand_merge(high_mask, high_merge))
17477 }
17478 );
17479 kernel(self, values, mask, merge)
17480 }
17481 #[inline(always)]
17482 fn split_u64x8(self, a: u64x8<Self>) -> (u64x4<Self>, u64x4<Self>) {
17483 (
17484 u64x4 {
17485 val: crate::support::Aligned256(a.val.0[0]),
17486 simd: self,
17487 },
17488 u64x4 {
17489 val: crate::support::Aligned256(a.val.0[1]),
17490 simd: self,
17491 },
17492 )
17493 }
17494 #[inline(always)]
17495 fn from_bitmask_mask64x8(self, bits: u64) -> mask64x8<Self> {
17496 crate::kernel!(
17497 #[inline(always)]
17498 fn kernel(token: Avx2, bits: u64) -> mask64x8<Avx2> {
17499 {
17500 let bit_lanes = _mm256_set1_epi64x(bits.cast_signed());
17501 mask64x8 {
17502 val: crate::support::Aligned512([
17503 {
17504 let bit_mask = _mm256_set_epi64x(8, 4, 2, 1);
17505 _mm256_cmpeq_epi64(_mm256_and_si256(bit_lanes, bit_mask), bit_mask)
17506 },
17507 {
17508 let bit_mask = _mm256_set_epi64x(128, 64, 32, 16);
17509 _mm256_cmpeq_epi64(_mm256_and_si256(bit_lanes, bit_mask), bit_mask)
17510 },
17511 ]),
17512 simd: token,
17513 }
17514 }
17515 }
17516 );
17517 kernel(self, bits)
17518 }
17519 #[inline(always)]
17520 fn from_vector_mask64x8(self, vector: i64x8<Self>) -> mask64x8<Self> {
17521 mask64x8 {
17522 val: vector.val,
17523 simd: self,
17524 }
17525 }
17526 #[inline(always)]
17527 fn to_vector_mask64x8(self, a: mask64x8<Self>) -> i64x8<Self> {
17528 i64x8 {
17529 val: a.val,
17530 simd: self,
17531 }
17532 }
17533 #[inline(always)]
17534 fn set_mask64x8(self, a: &mut mask64x8<Self>, index: usize, value: bool) -> () {
17535 assert!(
17536 index < 8usize,
17537 "mask lane index {index} is out of bounds for {} lanes",
17538 8usize
17539 );
17540 let mut lanes: [i64; 8usize] = (*a).into();
17541 lanes[index] = if value { !0 } else { 0 };
17542 *a = lanes.simd_into(self);
17543 }
17544 #[inline(always)]
17545 fn rotate_elements_left_mask64x8<const OFFSET: usize>(
17546 self,
17547 a: mask64x8<Self>,
17548 ) -> mask64x8<Self> {
17549 let int = i64x8 {
17550 val: crate::transmute::checked_transmute_copy(&a.val),
17551 simd: self,
17552 };
17553 let rotated = int.rotate_elements_left::<OFFSET>();
17554 mask64x8 {
17555 val: crate::transmute::checked_transmute_copy(&rotated.val),
17556 simd: self,
17557 }
17558 }
17559 #[inline(always)]
17560 fn rotate_elements_right_mask64x8<const OFFSET: usize>(
17561 self,
17562 a: mask64x8<Self>,
17563 ) -> mask64x8<Self> {
17564 let int = i64x8 {
17565 val: crate::transmute::checked_transmute_copy(&a.val),
17566 simd: self,
17567 };
17568 let rotated = int.rotate_elements_right::<OFFSET>();
17569 mask64x8 {
17570 val: crate::transmute::checked_transmute_copy(&rotated.val),
17571 simd: self,
17572 }
17573 }
17574 #[inline(always)]
17575 fn split_mask64x8(self, a: mask64x8<Self>) -> (mask64x4<Self>, mask64x4<Self>) {
17576 (
17577 mask64x4 {
17578 val: crate::support::Aligned256(a.val.0[0]),
17579 simd: self,
17580 },
17581 mask64x4 {
17582 val: crate::support::Aligned256(a.val.0[1]),
17583 simd: self,
17584 },
17585 )
17586 }
17587}
17588impl<S: Simd> SimdFrom<__m256, S> for f32x8<S> {
17589 #[inline(always)]
17590 fn simd_from(simd: S, arch: __m256) -> Self {
17591 Self {
17592 val: crate::transmute::checked_transmute_copy(&arch),
17593 simd,
17594 }
17595 }
17596}
17597impl<S: Simd> From<f32x8<S>> for __m256 {
17598 #[inline(always)]
17599 fn from(value: f32x8<S>) -> Self {
17600 crate::transmute::checked_transmute_copy(&value.val)
17601 }
17602}
17603impl<S: Simd> SimdFrom<__m256i, S> for i8x32<S> {
17604 #[inline(always)]
17605 fn simd_from(simd: S, arch: __m256i) -> Self {
17606 Self {
17607 val: crate::transmute::checked_transmute_copy(&arch),
17608 simd,
17609 }
17610 }
17611}
17612impl<S: Simd> From<i8x32<S>> for __m256i {
17613 #[inline(always)]
17614 fn from(value: i8x32<S>) -> Self {
17615 crate::transmute::checked_transmute_copy(&value.val)
17616 }
17617}
17618impl<S: Simd> SimdFrom<__m256i, S> for u8x32<S> {
17619 #[inline(always)]
17620 fn simd_from(simd: S, arch: __m256i) -> Self {
17621 Self {
17622 val: crate::transmute::checked_transmute_copy(&arch),
17623 simd,
17624 }
17625 }
17626}
17627impl<S: Simd> From<u8x32<S>> for __m256i {
17628 #[inline(always)]
17629 fn from(value: u8x32<S>) -> Self {
17630 crate::transmute::checked_transmute_copy(&value.val)
17631 }
17632}
17633impl<S: Simd> SimdFrom<__m256i, S> for mask8x32<S> {
17634 #[inline(always)]
17635 fn simd_from(simd: S, arch: __m256i) -> Self {
17636 let lanes: [i8; 32usize] = crate::transmute::checked_transmute_copy(&arch);
17637 lanes.simd_into(simd)
17638 }
17639}
17640impl<S: Simd> From<mask8x32<S>> for __m256i {
17641 #[inline(always)]
17642 fn from(value: mask8x32<S>) -> Self {
17643 let lanes: [i8; 32usize] = value.into();
17644 crate::transmute::checked_transmute_copy(&lanes)
17645 }
17646}
17647impl<S: Simd> SimdFrom<__m256i, S> for i16x16<S> {
17648 #[inline(always)]
17649 fn simd_from(simd: S, arch: __m256i) -> Self {
17650 Self {
17651 val: crate::transmute::checked_transmute_copy(&arch),
17652 simd,
17653 }
17654 }
17655}
17656impl<S: Simd> From<i16x16<S>> for __m256i {
17657 #[inline(always)]
17658 fn from(value: i16x16<S>) -> Self {
17659 crate::transmute::checked_transmute_copy(&value.val)
17660 }
17661}
17662impl<S: Simd> SimdFrom<__m256i, S> for u16x16<S> {
17663 #[inline(always)]
17664 fn simd_from(simd: S, arch: __m256i) -> Self {
17665 Self {
17666 val: crate::transmute::checked_transmute_copy(&arch),
17667 simd,
17668 }
17669 }
17670}
17671impl<S: Simd> From<u16x16<S>> for __m256i {
17672 #[inline(always)]
17673 fn from(value: u16x16<S>) -> Self {
17674 crate::transmute::checked_transmute_copy(&value.val)
17675 }
17676}
17677impl<S: Simd> SimdFrom<__m256i, S> for mask16x16<S> {
17678 #[inline(always)]
17679 fn simd_from(simd: S, arch: __m256i) -> Self {
17680 let lanes: [i16; 16usize] = crate::transmute::checked_transmute_copy(&arch);
17681 lanes.simd_into(simd)
17682 }
17683}
17684impl<S: Simd> From<mask16x16<S>> for __m256i {
17685 #[inline(always)]
17686 fn from(value: mask16x16<S>) -> Self {
17687 let lanes: [i16; 16usize] = value.into();
17688 crate::transmute::checked_transmute_copy(&lanes)
17689 }
17690}
17691impl<S: Simd> SimdFrom<__m256i, S> for i32x8<S> {
17692 #[inline(always)]
17693 fn simd_from(simd: S, arch: __m256i) -> Self {
17694 Self {
17695 val: crate::transmute::checked_transmute_copy(&arch),
17696 simd,
17697 }
17698 }
17699}
17700impl<S: Simd> From<i32x8<S>> for __m256i {
17701 #[inline(always)]
17702 fn from(value: i32x8<S>) -> Self {
17703 crate::transmute::checked_transmute_copy(&value.val)
17704 }
17705}
17706impl<S: Simd> SimdFrom<__m256i, S> for u32x8<S> {
17707 #[inline(always)]
17708 fn simd_from(simd: S, arch: __m256i) -> Self {
17709 Self {
17710 val: crate::transmute::checked_transmute_copy(&arch),
17711 simd,
17712 }
17713 }
17714}
17715impl<S: Simd> From<u32x8<S>> for __m256i {
17716 #[inline(always)]
17717 fn from(value: u32x8<S>) -> Self {
17718 crate::transmute::checked_transmute_copy(&value.val)
17719 }
17720}
17721impl<S: Simd> SimdFrom<__m256i, S> for mask32x8<S> {
17722 #[inline(always)]
17723 fn simd_from(simd: S, arch: __m256i) -> Self {
17724 let lanes: [i32; 8usize] = crate::transmute::checked_transmute_copy(&arch);
17725 lanes.simd_into(simd)
17726 }
17727}
17728impl<S: Simd> From<mask32x8<S>> for __m256i {
17729 #[inline(always)]
17730 fn from(value: mask32x8<S>) -> Self {
17731 let lanes: [i32; 8usize] = value.into();
17732 crate::transmute::checked_transmute_copy(&lanes)
17733 }
17734}
17735impl<S: Simd> SimdFrom<__m256d, S> for f64x4<S> {
17736 #[inline(always)]
17737 fn simd_from(simd: S, arch: __m256d) -> Self {
17738 Self {
17739 val: crate::transmute::checked_transmute_copy(&arch),
17740 simd,
17741 }
17742 }
17743}
17744impl<S: Simd> From<f64x4<S>> for __m256d {
17745 #[inline(always)]
17746 fn from(value: f64x4<S>) -> Self {
17747 crate::transmute::checked_transmute_copy(&value.val)
17748 }
17749}
17750impl<S: Simd> SimdFrom<__m256i, S> for i64x4<S> {
17751 #[inline(always)]
17752 fn simd_from(simd: S, arch: __m256i) -> Self {
17753 Self {
17754 val: crate::transmute::checked_transmute_copy(&arch),
17755 simd,
17756 }
17757 }
17758}
17759impl<S: Simd> From<i64x4<S>> for __m256i {
17760 #[inline(always)]
17761 fn from(value: i64x4<S>) -> Self {
17762 crate::transmute::checked_transmute_copy(&value.val)
17763 }
17764}
17765impl<S: Simd> SimdFrom<__m256i, S> for u64x4<S> {
17766 #[inline(always)]
17767 fn simd_from(simd: S, arch: __m256i) -> Self {
17768 Self {
17769 val: crate::transmute::checked_transmute_copy(&arch),
17770 simd,
17771 }
17772 }
17773}
17774impl<S: Simd> From<u64x4<S>> for __m256i {
17775 #[inline(always)]
17776 fn from(value: u64x4<S>) -> Self {
17777 crate::transmute::checked_transmute_copy(&value.val)
17778 }
17779}
17780impl<S: Simd> SimdFrom<__m256i, S> for mask64x4<S> {
17781 #[inline(always)]
17782 fn simd_from(simd: S, arch: __m256i) -> Self {
17783 let lanes: [i64; 4usize] = crate::transmute::checked_transmute_copy(&arch);
17784 lanes.simd_into(simd)
17785 }
17786}
17787impl<S: Simd> From<mask64x4<S>> for __m256i {
17788 #[inline(always)]
17789 fn from(value: mask64x4<S>) -> Self {
17790 let lanes: [i64; 4usize] = value.into();
17791 crate::transmute::checked_transmute_copy(&lanes)
17792 }
17793}
17794crate::kernel!(
17795 #[doc = r" This is a version of the `alignr` intrinsic that takes a non-const shift argument. The shift is still"]
17796 #[doc = r" expected to be constant in practice, so the match statement will be optimized out. This exists because"]
17797 #[doc = r" Rust doesn't currently let you do math on const generics."]
17798 #[inline(always)]
17799 fn dyn_alignr_128(token: Avx2, a: __m128i, b: __m128i, shift: usize) -> __m128i {
17800 match shift {
17801 0usize => _mm_alignr_epi8::<0i32>(a, b),
17802 1usize => _mm_alignr_epi8::<1i32>(a, b),
17803 2usize => _mm_alignr_epi8::<2i32>(a, b),
17804 3usize => _mm_alignr_epi8::<3i32>(a, b),
17805 4usize => _mm_alignr_epi8::<4i32>(a, b),
17806 5usize => _mm_alignr_epi8::<5i32>(a, b),
17807 6usize => _mm_alignr_epi8::<6i32>(a, b),
17808 7usize => _mm_alignr_epi8::<7i32>(a, b),
17809 8usize => _mm_alignr_epi8::<8i32>(a, b),
17810 9usize => _mm_alignr_epi8::<9i32>(a, b),
17811 10usize => _mm_alignr_epi8::<10i32>(a, b),
17812 11usize => _mm_alignr_epi8::<11i32>(a, b),
17813 12usize => _mm_alignr_epi8::<12i32>(a, b),
17814 13usize => _mm_alignr_epi8::<13i32>(a, b),
17815 14usize => _mm_alignr_epi8::<14i32>(a, b),
17816 15usize => _mm_alignr_epi8::<15i32>(a, b),
17817 _ => unreachable!(),
17818 }
17819 }
17820);
17821crate::kernel!(
17822 #[doc = r" This is a version of the `alignr` intrinsic that takes a non-const shift argument. The shift is still"]
17823 #[doc = r" expected to be constant in practice, so the match statement will be optimized out. This exists because"]
17824 #[doc = r" Rust doesn't currently let you do math on const generics."]
17825 #[inline(always)]
17826 fn dyn_alignr_256(token: Avx2, a: __m256i, b: __m256i, shift: usize) -> __m256i {
17827 match shift {
17828 0usize => _mm256_alignr_epi8::<0i32>(a, b),
17829 1usize => _mm256_alignr_epi8::<1i32>(a, b),
17830 2usize => _mm256_alignr_epi8::<2i32>(a, b),
17831 3usize => _mm256_alignr_epi8::<3i32>(a, b),
17832 4usize => _mm256_alignr_epi8::<4i32>(a, b),
17833 5usize => _mm256_alignr_epi8::<5i32>(a, b),
17834 6usize => _mm256_alignr_epi8::<6i32>(a, b),
17835 7usize => _mm256_alignr_epi8::<7i32>(a, b),
17836 8usize => _mm256_alignr_epi8::<8i32>(a, b),
17837 9usize => _mm256_alignr_epi8::<9i32>(a, b),
17838 10usize => _mm256_alignr_epi8::<10i32>(a, b),
17839 11usize => _mm256_alignr_epi8::<11i32>(a, b),
17840 12usize => _mm256_alignr_epi8::<12i32>(a, b),
17841 13usize => _mm256_alignr_epi8::<13i32>(a, b),
17842 14usize => _mm256_alignr_epi8::<14i32>(a, b),
17843 15usize => _mm256_alignr_epi8::<15i32>(a, b),
17844 _ => unreachable!(),
17845 }
17846 }
17847);
17848crate::kernel!(
17849 #[doc = r" Computes one output __m256i for `cross_block_alignr_*` operations."]
17850 #[doc = r""]
17851 #[doc = r" Given an array of registers, each containing two 128-bit blocks, extracts two adjacent blocks (`lo_idx` and"]
17852 #[doc = r" `hi_idx` = `lo_idx + 1`) and performs `alignr` with `intra_shift`."]
17853 #[inline(always)]
17854 fn cross_block_alignr_one(
17855 token: Avx2,
17856 regs: &[__m256i],
17857 block_idx: usize,
17858 shift_bytes: usize,
17859 ) -> __m256i {
17860 let lo_idx = block_idx + (shift_bytes / 16);
17861 let intra_shift = shift_bytes % 16;
17862 let lo_blocks = if lo_idx & 1 == 0 {
17863 regs[lo_idx / 2]
17864 } else {
17865 _mm256_permute2x128_si256::<0x21>(regs[lo_idx / 2], regs[(lo_idx / 2) + 1])
17866 };
17867 let hi_idx = lo_idx + 1;
17868 let hi_blocks = if hi_idx & 1 == 0 {
17869 regs[hi_idx / 2]
17870 } else {
17871 _mm256_permute2x128_si256::<0x21>(regs[hi_idx / 2], regs[(hi_idx / 2) + 1])
17872 };
17873 dyn_alignr_256(token, hi_blocks, lo_blocks, intra_shift)
17874 }
17875);
17876crate::kernel!(
17877 #[doc = r" Concatenates `b` and `a` (each 2 x __m256i = 4 blocks) and extracts 4 blocks starting at byte offset"]
17878 #[doc = r" `shift_bytes`. Extracts from [b : a] (b in low bytes, a in high bytes), matching alignr semantics."]
17879 #[inline(always)]
17880 fn cross_block_alignr_256x2(
17881 token: Avx2,
17882 a: [__m256i; 2],
17883 b: [__m256i; 2],
17884 shift_bytes: usize,
17885 ) -> [__m256i; 2] {
17886 let regs = [b[0], b[1], a[0], a[1]];
17887 [
17888 cross_block_alignr_one(token, ®s, 0, shift_bytes),
17889 cross_block_alignr_one(token, ®s, 2, shift_bytes),
17890 ]
17891 }
17892);
17893crate::kernel!(
17894 #[doc = r" Concatenates `b` and `a` (each 1 x __m256i = 2 blocks) and extracts 2 blocks starting at byte offset"]
17895 #[doc = r" `shift_bytes`. Extracts from [b : a] (b in low bytes, a in high bytes), matching alignr semantics."]
17896 #[inline(always)]
17897 fn cross_block_alignr_256x1(
17898 token: Avx2,
17899 a: __m256i,
17900 b: __m256i,
17901 shift_bytes: usize,
17902 ) -> __m256i {
17903 let regs = [b, a];
17904 cross_block_alignr_one(token, ®s, 0, shift_bytes)
17905 }
17906);