1#[cfg(all(feature = "libm", not(feature = "std")))]
7#[allow(
8 unused_imports,
9 reason = "Generated backends use different subsets of these helpers"
10)]
11use crate::math::FloatExt as _;
12use crate::{Level, arch_types::ArchTypes, prelude::*, seal::Seal};
13use crate::{
14 f32x4, f32x8, f32x16, f64x2, f64x4, f64x8, i8x16, i8x32, i8x64, i16x8, i16x16, i16x32, i32x4,
15 i32x8, i32x16, i64x2, i64x4, i64x8, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16,
16 mask16x32, mask32x4, mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64,
17 u16x8, u16x16, u16x32, u32x4, u32x8, u32x16, u64x2, u64x4, u64x8,
18};
19#[cfg(target_arch = "x86")]
20use core::arch::x86::*;
21#[cfg(target_arch = "x86_64")]
22use core::arch::x86_64::*;
23use core::ops::*;
24#[doc = "A token for SSE4.2 intrinsics on `x86` and `x86_64`, representing the x86-64-v2 level."]
25#[doc = "# Browsing the documentation"]
26#[doc = "The method list on this struct is very verbose."]
27#[doc = "A better way to browse the docs is by looking at vector types such as [`u32x4`], [`f32x4`] or [`mask32x4`]."]
28#[doc = "They include all the operations listed here, and also provide some additional convenience methods."]
29#[derive(Clone, Copy, Debug)]
30pub struct Sse4_2 {
31 _private: (),
32}
33impl Sse4_2 {
34 #[doc = "Create a SIMD token proving that the x86-64-v2 features are available."]
35 #[doc = r""]
36 #[doc = r" Most users should safely obtain the token from [`Level::new`] instead of calling this function."]
37 #[doc = r""]
38 #[doc = r" This function can be called without an `unsafe` block from a function"]
39 #[doc = r" with all the required target features enabled via the `#[target_feature]` annotation."]
40 #[doc = r""]
41 #[doc = r" # Safety"]
42 #[doc = r""]
43 #[doc = "When invoking this function through an `unsafe` block, the caller must ensure that the current CPU supports `fxsr`, `sse4.2`, `cmpxchg16b`, `popcnt`."]
44 #[inline]
45 #[target_feature(enable = "fxsr,sse4.2,cmpxchg16b,popcnt")]
46 pub const fn assume_supported() -> Self {
47 Self { _private: () }
48 }
49}
50impl Seal for Sse4_2 {}
51impl ExtractToken for Sse4_2 {
52 type S = Self;
53 #[inline]
54 fn token(&self) -> Self {
55 *self
56 }
57}
58impl ArchTypes for Sse4_2 {
59 type f32x4 = crate::support::Aligned128<__m128>;
60 type i8x16 = crate::support::Aligned128<__m128i>;
61 type u8x16 = crate::support::Aligned128<__m128i>;
62 type mask8x16 = crate::support::Aligned128<__m128i>;
63 type i16x8 = crate::support::Aligned128<__m128i>;
64 type u16x8 = crate::support::Aligned128<__m128i>;
65 type mask16x8 = crate::support::Aligned128<__m128i>;
66 type i32x4 = crate::support::Aligned128<__m128i>;
67 type u32x4 = crate::support::Aligned128<__m128i>;
68 type mask32x4 = crate::support::Aligned128<__m128i>;
69 type f64x2 = crate::support::Aligned128<__m128d>;
70 type i64x2 = crate::support::Aligned128<__m128i>;
71 type u64x2 = crate::support::Aligned128<__m128i>;
72 type mask64x2 = crate::support::Aligned128<__m128i>;
73 type f32x8 = crate::support::Aligned256<[__m128; 2usize]>;
74 type i8x32 = crate::support::Aligned256<[__m128i; 2usize]>;
75 type u8x32 = crate::support::Aligned256<[__m128i; 2usize]>;
76 type mask8x32 = crate::support::Aligned256<[__m128i; 2usize]>;
77 type i16x16 = crate::support::Aligned256<[__m128i; 2usize]>;
78 type u16x16 = crate::support::Aligned256<[__m128i; 2usize]>;
79 type mask16x16 = crate::support::Aligned256<[__m128i; 2usize]>;
80 type i32x8 = crate::support::Aligned256<[__m128i; 2usize]>;
81 type u32x8 = crate::support::Aligned256<[__m128i; 2usize]>;
82 type mask32x8 = crate::support::Aligned256<[__m128i; 2usize]>;
83 type f64x4 = crate::support::Aligned256<[__m128d; 2usize]>;
84 type i64x4 = crate::support::Aligned256<[__m128i; 2usize]>;
85 type u64x4 = crate::support::Aligned256<[__m128i; 2usize]>;
86 type mask64x4 = crate::support::Aligned256<[__m128i; 2usize]>;
87 type f32x16 = crate::support::Aligned512<[__m128; 4usize]>;
88 type i8x64 = crate::support::Aligned512<[__m128i; 4usize]>;
89 type u8x64 = crate::support::Aligned512<[__m128i; 4usize]>;
90 type mask8x64 = crate::support::Aligned512<[__m128i; 4usize]>;
91 type i16x32 = crate::support::Aligned512<[__m128i; 4usize]>;
92 type u16x32 = crate::support::Aligned512<[__m128i; 4usize]>;
93 type mask16x32 = crate::support::Aligned512<[__m128i; 4usize]>;
94 type i32x16 = crate::support::Aligned512<[__m128i; 4usize]>;
95 type u32x16 = crate::support::Aligned512<[__m128i; 4usize]>;
96 type mask32x16 = crate::support::Aligned512<[__m128i; 4usize]>;
97 type f64x8 = crate::support::Aligned512<[__m128d; 4usize]>;
98 type i64x8 = crate::support::Aligned512<[__m128i; 4usize]>;
99 type u64x8 = crate::support::Aligned512<[__m128i; 4usize]>;
100 type mask64x8 = crate::support::Aligned512<[__m128i; 4usize]>;
101}
102impl Simd for Sse4_2 {
103 type f32s = f32x4<Self>;
104 type f64s = f64x2<Self>;
105 type u8s = u8x16<Self>;
106 type i8s = i8x16<Self>;
107 type u16s = u16x8<Self>;
108 type i16s = i16x8<Self>;
109 type u32s = u32x4<Self>;
110 type i32s = i32x4<Self>;
111 type u64s = u64x2<Self>;
112 type i64s = i64x2<Self>;
113 type mask8s = mask8x16<Self>;
114 type mask16s = mask16x8<Self>;
115 type mask32s = mask32x4<Self>;
116 type mask64s = mask64x2<Self>;
117 #[inline(always)]
118 fn level(self) -> Level {
119 Level::Sse4_2(self)
120 }
121 #[inline]
122 fn vectorize<F: FnOnce() -> R, R>(self, f: F) -> R {
123 #[inline]
124 #[target_feature(enable = "fxsr,sse4.2,cmpxchg16b,popcnt")]
125 fn vectorize_sse4_2<F: FnOnce() -> R, R>(f: F) -> R {
126 f()
127 }
128 unsafe { vectorize_sse4_2(f) }
129 }
130 #[inline(always)]
131 fn abs_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
132 crate::kernel!(
133 #[inline(always)]
134 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
135 _mm_andnot_ps(_mm_set1_ps(-0.0), a.into()).simd_into(token)
136 }
137 );
138 kernel(self, a)
139 }
140 #[inline(always)]
141 fn splat_f32x4(self, val: f32) -> f32x4<Self> {
142 crate::kernel!(
143 #[inline(always)]
144 fn kernel(token: Sse4_2, val: f32) -> f32x4<Sse4_2> {
145 _mm_set1_ps(val).simd_into(token)
146 }
147 );
148 kernel(self, val)
149 }
150 #[inline(always)]
151 fn reverse_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
152 let indices: u8x16<Self> =
153 [12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3].simd_into(self);
154 self.swizzle_dyn_f32x4(a, indices)
155 }
156 #[inline(always)]
157 fn slide_f32x4<const SHIFT: usize>(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
158 if SHIFT >= 4usize {
159 return b;
160 }
161 let result = dyn_alignr_128(
162 self,
163 Bytes::to_bytes(b).val.0,
164 Bytes::to_bytes(a).val.0,
165 SHIFT * 4usize,
166 );
167 Bytes::from_bytes(u8x16 {
168 val: crate::support::Aligned128(result),
169 simd: self,
170 })
171 }
172 #[inline(always)]
173 fn compress_f32x4(self, values: f32x4<Self>, mask: mask32x4<Self>) -> f32x4<Self> {
174 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
175 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0);
176 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
177 }
178 #[inline(always)]
179 fn compress_merge_f32x4(
180 self,
181 values: f32x4<Self>,
182 mask: mask32x4<Self>,
183 merge: f32x4<Self>,
184 ) -> f32x4<Self> {
185 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
186 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0);
187 let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control);
188 let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0));
189 Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed))
190 }
191 #[inline(always)]
192 fn expand_f32x4(self, values: f32x4<Self>, mask: mask32x4<Self>) -> f32x4<Self> {
193 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
194 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0);
195 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
196 }
197 #[inline(always)]
198 fn expand_merge_f32x4(
199 self,
200 values: f32x4<Self>,
201 mask: mask32x4<Self>,
202 merge: f32x4<Self>,
203 ) -> f32x4<Self> {
204 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
205 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0);
206 let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control));
207 self.select_f32x4(mask, expanded, merge)
208 }
209 #[inline(always)]
210 fn neg_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
211 crate::kernel!(
212 #[inline(always)]
213 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
214 _mm_xor_ps(a.into(), _mm_set1_ps(-0.0)).simd_into(token)
215 }
216 );
217 kernel(self, a)
218 }
219 #[inline(always)]
220 fn sqrt_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
221 crate::kernel!(
222 #[inline(always)]
223 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
224 _mm_sqrt_ps(a.into()).simd_into(token)
225 }
226 );
227 kernel(self, a)
228 }
229 #[inline(always)]
230 fn approximate_recip_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
231 crate::kernel!(
232 #[inline(always)]
233 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
234 _mm_rcp_ps(a.into()).simd_into(token)
235 }
236 );
237 kernel(self, a)
238 }
239 #[inline(always)]
240 fn add_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
241 crate::kernel!(
242 #[inline(always)]
243 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
244 _mm_add_ps(a.into(), b.into()).simd_into(token)
245 }
246 );
247 kernel(self, a, b)
248 }
249 #[inline(always)]
250 fn sub_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
251 crate::kernel!(
252 #[inline(always)]
253 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
254 _mm_sub_ps(a.into(), b.into()).simd_into(token)
255 }
256 );
257 kernel(self, a, b)
258 }
259 #[inline(always)]
260 fn mul_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
261 crate::kernel!(
262 #[inline(always)]
263 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
264 _mm_mul_ps(a.into(), b.into()).simd_into(token)
265 }
266 );
267 kernel(self, a, b)
268 }
269 #[inline(always)]
270 fn div_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
271 crate::kernel!(
272 #[inline(always)]
273 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
274 _mm_div_ps(a.into(), b.into()).simd_into(token)
275 }
276 );
277 kernel(self, a, b)
278 }
279 #[inline(always)]
280 fn copysign_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
281 crate::kernel!(
282 #[inline(always)]
283 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
284 let mask = _mm_set1_ps(-0.0);
285 _mm_or_ps(_mm_and_ps(mask, b.into()), _mm_andnot_ps(mask, a.into()))
286 .simd_into(token)
287 }
288 );
289 kernel(self, a, b)
290 }
291 #[inline(always)]
292 fn reduce_max_f32x4(self, a: f32x4<Self>) -> f32 {
293 crate::kernel!(
294 #[inline(always)]
295 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32 {
296 let reduced: __m128 = a.into();
297 let shifted = _mm_castsi128_ps(_mm_srli_si128::<8>(_mm_castps_si128(reduced)));
298 let reduced = { _mm_max_ps(reduced, shifted) };
299 let shifted = _mm_castsi128_ps(_mm_srli_si128::<4>(_mm_castps_si128(reduced)));
300 let reduced = { _mm_max_ps(reduced, shifted) };
301 _mm_cvtss_f32(reduced)
302 }
303 );
304 kernel(self, a)
305 }
306 #[inline(always)]
307 fn reduce_min_f32x4(self, a: f32x4<Self>) -> f32 {
308 crate::kernel!(
309 #[inline(always)]
310 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32 {
311 let reduced: __m128 = a.into();
312 let shifted = _mm_castsi128_ps(_mm_srli_si128::<8>(_mm_castps_si128(reduced)));
313 let reduced = { _mm_min_ps(reduced, shifted) };
314 let shifted = _mm_castsi128_ps(_mm_srli_si128::<4>(_mm_castps_si128(reduced)));
315 let reduced = { _mm_min_ps(reduced, shifted) };
316 _mm_cvtss_f32(reduced)
317 }
318 );
319 kernel(self, a)
320 }
321 #[inline(always)]
322 fn reduce_max_precise_f32x4(self, a: f32x4<Self>) -> f32 {
323 crate::kernel!(
324 #[inline(always)]
325 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32 {
326 let reduced: __m128 = a.into();
327 let shifted = _mm_castsi128_ps(_mm_srli_si128::<8>(_mm_castps_si128(reduced)));
328 let reduced = {
329 let intermediate = _mm_max_ps(reduced, shifted);
330 let b_is_nan = _mm_cmpunord_ps(shifted, shifted);
331 _mm_blendv_ps(intermediate, reduced, b_is_nan)
332 };
333 let shifted = _mm_castsi128_ps(_mm_srli_si128::<4>(_mm_castps_si128(reduced)));
334 let reduced = {
335 let intermediate = _mm_max_ps(reduced, shifted);
336 let b_is_nan = _mm_cmpunord_ps(shifted, shifted);
337 _mm_blendv_ps(intermediate, reduced, b_is_nan)
338 };
339 _mm_cvtss_f32(reduced)
340 }
341 );
342 kernel(self, a)
343 }
344 #[inline(always)]
345 fn reduce_min_precise_f32x4(self, a: f32x4<Self>) -> f32 {
346 crate::kernel!(
347 #[inline(always)]
348 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32 {
349 let reduced: __m128 = a.into();
350 let shifted = _mm_castsi128_ps(_mm_srli_si128::<8>(_mm_castps_si128(reduced)));
351 let reduced = {
352 let intermediate = _mm_min_ps(reduced, shifted);
353 let b_is_nan = _mm_cmpunord_ps(shifted, shifted);
354 _mm_blendv_ps(intermediate, reduced, b_is_nan)
355 };
356 let shifted = _mm_castsi128_ps(_mm_srli_si128::<4>(_mm_castps_si128(reduced)));
357 let reduced = {
358 let intermediate = _mm_min_ps(reduced, shifted);
359 let b_is_nan = _mm_cmpunord_ps(shifted, shifted);
360 _mm_blendv_ps(intermediate, reduced, b_is_nan)
361 };
362 _mm_cvtss_f32(reduced)
363 }
364 );
365 kernel(self, a)
366 }
367 #[inline(always)]
368 fn reduce_sum_f32x4(self, a: f32x4<Self>) -> f32 {
369 crate::kernel!(
370 #[inline(always)]
371 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32 {
372 let a: __m128 = a.into();
373 let adjacent = _mm_add_ps(a, _mm_shuffle_ps::<0b10_11_00_01>(a, a));
374 _mm_cvtss_f32(_mm_add_ss(adjacent, _mm_movehl_ps(adjacent, adjacent)))
375 }
376 );
377 kernel(self, a)
378 }
379 #[inline(always)]
380 fn reduce_product_f32x4(self, a: f32x4<Self>) -> f32 {
381 crate::kernel!(
382 #[inline(always)]
383 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32 {
384 let a: __m128 = a.into();
385 let adjacent = _mm_mul_ps(a, _mm_shuffle_ps::<0b10_11_00_01>(a, a));
386 _mm_cvtss_f32(_mm_mul_ss(adjacent, _mm_movehl_ps(adjacent, adjacent)))
387 }
388 );
389 kernel(self, a)
390 }
391 #[inline(always)]
392 fn max_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
393 crate::kernel!(
394 #[inline(always)]
395 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
396 _mm_max_ps(a.into(), b.into()).simd_into(token)
397 }
398 );
399 kernel(self, a, b)
400 }
401 #[inline(always)]
402 fn min_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
403 crate::kernel!(
404 #[inline(always)]
405 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
406 _mm_min_ps(a.into(), b.into()).simd_into(token)
407 }
408 );
409 kernel(self, a, b)
410 }
411 #[inline(always)]
412 fn max_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
413 crate::kernel!(
414 #[inline(always)]
415 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
416 let intermediate = _mm_max_ps(a.into(), b.into());
417 let b_is_nan = _mm_cmpunord_ps(b.into(), b.into());
418 _mm_blendv_ps(intermediate, a.into(), b_is_nan).simd_into(token)
419 }
420 );
421 kernel(self, a, b)
422 }
423 #[inline(always)]
424 fn min_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
425 crate::kernel!(
426 #[inline(always)]
427 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
428 let intermediate = _mm_min_ps(a.into(), b.into());
429 let b_is_nan = _mm_cmpunord_ps(b.into(), b.into());
430 _mm_blendv_ps(intermediate, a.into(), b_is_nan).simd_into(token)
431 }
432 );
433 kernel(self, a, b)
434 }
435 #[inline(always)]
436 fn clamp_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
437 crate::kernel!(
438 #[inline(always)]
439 fn kernel(
440 token: Sse4_2,
441 a: f32x4<Sse4_2>,
442 b: f32x4<Sse4_2>,
443 c: f32x4<Sse4_2>,
444 ) -> f32x4<Sse4_2> {
445 debug_assert!(
446 b.simd_le(c).all_true(),
447 "clamp requires min <= max for all elements"
448 );
449 _mm_min_ps(c.into(), _mm_max_ps(b.into(), a.into())).simd_into(token)
450 }
451 );
452 kernel(self, a, b, c)
453 }
454 #[inline(always)]
455 fn simd_eq_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
456 crate::kernel!(
457 #[inline(always)]
458 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> mask32x4<Sse4_2> {
459 _mm_castps_si128(_mm_cmpeq_ps(a.into(), b.into())).simd_into(token)
460 }
461 );
462 kernel(self, a, b)
463 }
464 #[inline(always)]
465 fn simd_ne_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
466 crate::kernel!(
467 #[inline(always)]
468 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> mask32x4<Sse4_2> {
469 _mm_castps_si128(_mm_cmpneq_ps(a.into(), b.into())).simd_into(token)
470 }
471 );
472 kernel(self, a, b)
473 }
474 #[inline(always)]
475 fn simd_lt_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
476 crate::kernel!(
477 #[inline(always)]
478 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> mask32x4<Sse4_2> {
479 _mm_castps_si128(_mm_cmplt_ps(a.into(), b.into())).simd_into(token)
480 }
481 );
482 kernel(self, a, b)
483 }
484 #[inline(always)]
485 fn simd_le_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
486 crate::kernel!(
487 #[inline(always)]
488 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> mask32x4<Sse4_2> {
489 _mm_castps_si128(_mm_cmple_ps(a.into(), b.into())).simd_into(token)
490 }
491 );
492 kernel(self, a, b)
493 }
494 #[inline(always)]
495 fn zip_low_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
496 crate::kernel!(
497 #[inline(always)]
498 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
499 _mm_unpacklo_ps(a.into(), b.into()).simd_into(token)
500 }
501 );
502 kernel(self, a, b)
503 }
504 #[inline(always)]
505 fn zip_high_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
506 crate::kernel!(
507 #[inline(always)]
508 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
509 _mm_unpackhi_ps(a.into(), b.into()).simd_into(token)
510 }
511 );
512 kernel(self, a, b)
513 }
514 #[inline(always)]
515 fn unzip_low_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
516 crate::kernel!(
517 #[inline(always)]
518 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
519 _mm_shuffle_ps::<0b10_00_10_00>(a.into(), b.into()).simd_into(token)
520 }
521 );
522 kernel(self, a, b)
523 }
524 #[inline(always)]
525 fn unzip_high_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
526 crate::kernel!(
527 #[inline(always)]
528 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>, b: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
529 _mm_shuffle_ps::<0b11_01_11_01>(a.into(), b.into()).simd_into(token)
530 }
531 );
532 kernel(self, a, b)
533 }
534 #[inline(always)]
535 fn interleave_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> (f32x4<Self>, f32x4<Self>) {
536 (self.zip_low_f32x4(a, b), self.zip_high_f32x4(a, b))
537 }
538 #[inline(always)]
539 fn deinterleave_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> (f32x4<Self>, f32x4<Self>) {
540 (self.unzip_low_f32x4(a, b), self.unzip_high_f32x4(a, b))
541 }
542 #[inline(always)]
543 fn mul_add_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
544 a * b + c
545 }
546 #[inline(always)]
547 fn mul_add_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
548 crate::math::mul_add_precise_f32x4(self, a, b, c)
549 }
550 #[inline(always)]
551 fn mul_sub_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
552 a * b - c
553 }
554 #[inline(always)]
555 fn mul_sub_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
556 self.mul_add_precise_f32x4(a, b, -c)
557 }
558 #[inline(always)]
559 fn floor_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
560 crate::kernel!(
561 #[inline(always)]
562 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
563 _mm_round_ps::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
564 .simd_into(token)
565 }
566 );
567 kernel(self, a)
568 }
569 #[inline(always)]
570 fn ceil_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
571 crate::kernel!(
572 #[inline(always)]
573 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
574 _mm_round_ps::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
575 .simd_into(token)
576 }
577 );
578 kernel(self, a)
579 }
580 #[inline(always)]
581 fn round_ties_even_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
582 crate::kernel!(
583 #[inline(always)]
584 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
585 _mm_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
586 .simd_into(token)
587 }
588 );
589 kernel(self, a)
590 }
591 #[inline(always)]
592 fn round_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
593 let bias = self.splat_f32x4(const { f32::next_down(0.5) });
594 (a + self.copysign_f32x4(bias, a)).trunc()
595 }
596 #[inline(always)]
597 fn fract_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
598 a - self.trunc_f32x4(a)
599 }
600 #[inline(always)]
601 fn trunc_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
602 crate::kernel!(
603 #[inline(always)]
604 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> f32x4<Sse4_2> {
605 _mm_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
606 .simd_into(token)
607 }
608 );
609 kernel(self, a)
610 }
611 #[inline(always)]
612 fn to_degrees_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
613 a * const { f32::to_degrees(1.0) }
614 }
615 #[inline(always)]
616 fn to_radians_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
617 a * const { f32::to_radians(1.0) }
618 }
619 #[inline(always)]
620 fn is_nan_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
621 a.simd_ne(a)
622 }
623 #[inline(always)]
624 fn is_infinite_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
625 a.abs().simd_eq(f32::INFINITY)
626 }
627 #[inline(always)]
628 fn is_finite_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
629 a.abs().simd_lt(f32::INFINITY)
630 }
631 #[inline(always)]
632 fn is_subnormal_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
633 let i: i32x4<Self> = a.bitcast();
634 (i & 0x7F80_0000_i32).simd_eq(0) & (i & 0x007F_FFFF_i32).simd_ne(0)
635 }
636 #[inline(always)]
637 fn is_normal_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
638 let i: i32x4<Self> = a.bitcast();
639 let exp = i & 0x7F80_0000_i32;
640 exp.simd_ne(0) & exp.simd_ne(0x7F80_0000_i32)
641 }
642 #[inline(always)]
643 fn is_sign_positive_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
644 let i: i32x4<Self> = a.bitcast();
645 (i & i32::MIN).simd_eq(0)
646 }
647 #[inline(always)]
648 fn is_sign_negative_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
649 !a.is_sign_positive()
650 }
651 #[inline(always)]
652 fn select_f32x4(self, a: mask32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
653 crate::kernel!(
654 #[inline(always)]
655 fn kernel(
656 token: Sse4_2,
657 a: mask32x4<Sse4_2>,
658 b: f32x4<Sse4_2>,
659 c: f32x4<Sse4_2>,
660 ) -> f32x4<Sse4_2> {
661 _mm_blendv_ps(c.into(), b.into(), _mm_castsi128_ps(a.into())).simd_into(token)
662 }
663 );
664 kernel(self, a, b, c)
665 }
666 #[inline(always)]
667 fn combine_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x8<Self> {
668 f32x8 {
669 val: crate::support::Aligned256([a.val.0, b.val.0]),
670 simd: self,
671 }
672 }
673 #[inline(always)]
674 fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4<Self>; 4usize] {
675 crate::kernel!(
676 #[inline(always)]
677 fn kernel(token: Sse4_2, src: &[f32; 16usize]) -> [f32x4<Sse4_2>; 4usize] {
678 let (chunks, []) = src.as_chunks::<4usize>() else {
679 unreachable!()
680 };
681 let v0: __m128 =
682 crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[0]);
683 let v1: __m128 =
684 crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[1]);
685 let v2: __m128 =
686 crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[2]);
687 let v3: __m128 =
688 crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[3]);
689 let tmp0 = _mm_unpacklo_ps(v0, v1);
690 let tmp1 = _mm_unpackhi_ps(v0, v1);
691 let tmp2 = _mm_unpacklo_ps(v2, v3);
692 let tmp3 = _mm_unpackhi_ps(v2, v3);
693 let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2)));
694 let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2)));
695 let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3)));
696 let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3)));
697 [
698 out0.simd_into(token),
699 out1.simd_into(token),
700 out2.simd_into(token),
701 out3.simd_into(token),
702 ]
703 }
704 );
705 kernel(self, src)
706 }
707 #[inline(always)]
708 fn store_four_interleaved_f32x4(
709 self,
710 vectors: [f32x4<Self>; 4usize],
711 dest: &mut [f32; 16usize],
712 ) -> () {
713 crate::kernel!(
714 #[inline(always)]
715 fn kernel(
716 token: Sse4_2,
717 vectors: [f32x4<Sse4_2>; 4usize],
718 dest: &mut [f32; 16usize],
719 ) -> () {
720 let _ = token;
721 let v0: __m128 = vectors[0].into();
722 let v1: __m128 = vectors[1].into();
723 let v2: __m128 = vectors[2].into();
724 let v3: __m128 = vectors[3].into();
725 let tmp0 = _mm_unpacklo_ps(v0, v1);
726 let tmp1 = _mm_unpackhi_ps(v0, v1);
727 let tmp2 = _mm_unpacklo_ps(v2, v3);
728 let tmp3 = _mm_unpackhi_ps(v2, v3);
729 let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2)));
730 let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2)));
731 let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3)));
732 let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3)));
733 let (chunks, []) = dest.as_chunks_mut::<4usize>() else {
734 unreachable!()
735 };
736 crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>(
737 out0,
738 &mut chunks[0],
739 );
740 crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>(
741 out1,
742 &mut chunks[1],
743 );
744 crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>(
745 out2,
746 &mut chunks[2],
747 );
748 crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>(
749 out3,
750 &mut chunks[3],
751 );
752 }
753 );
754 kernel(self, vectors, dest);
755 }
756 #[inline(always)]
757 fn widen_f32x4(self, a: f32x4<Self>) -> (f64x2<Self>, f64x2<Self>) {
758 crate::kernel!(
759 #[inline(always)]
760 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> (f64x2<Sse4_2>, f64x2<Sse4_2>) {
761 let raw = a.into();
762 (
763 _mm_cvtps_pd(raw).simd_into(token),
764 _mm_cvtps_pd(_mm_movehl_ps(raw, raw)).simd_into(token),
765 )
766 }
767 );
768 kernel(self, a)
769 }
770 #[inline(always)]
771 fn cvt_u32_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
772 crate::kernel!(
773 #[inline(always)]
774 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> u32x4<Sse4_2> {
775 let mut converted = _mm_cvttps_epi32(a.into());
776 let in_range = _mm_cmplt_ps(a.into(), _mm_set1_ps(2147483648.0));
777 let all_in_range = _mm_movemask_ps(in_range) == 0b1111;
778 if !all_in_range {
779 let excess = _mm_sub_ps(a.into(), _mm_set1_ps(2147483648.0));
780 let excess_converted = _mm_cvttps_epi32(_mm_andnot_ps(in_range, excess));
781 converted = _mm_add_epi32(converted, excess_converted);
782 }
783 converted.simd_into(token)
784 }
785 );
786 kernel(self, a)
787 }
788 #[inline(always)]
789 fn cvt_u32_precise_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
790 crate::kernel!(
791 #[inline(always)]
792 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> u32x4<Sse4_2> {
793 let a = _mm_max_ps(a.into(), _mm_setzero_ps());
794 let mut converted = _mm_cvttps_epi32(a);
795 let in_range = _mm_cmplt_ps(a, _mm_set1_ps(2147483648.0));
796 let all_in_range = _mm_movemask_ps(in_range) == 0b1111;
797 if !all_in_range {
798 let exceeds_unsigned_range =
799 _mm_castps_si128(_mm_cmplt_ps(_mm_set1_ps(4294967040.0), a));
800 let excess = _mm_sub_ps(a, _mm_set1_ps(2147483648.0));
801 let excess_converted = _mm_cvttps_epi32(_mm_andnot_ps(in_range, excess));
802 converted = _mm_add_epi32(converted, excess_converted);
803 converted = _mm_or_si128(converted, exceeds_unsigned_range);
804 }
805 converted.simd_into(token)
806 }
807 );
808 kernel(self, a)
809 }
810 #[inline(always)]
811 fn cvt_i32_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
812 crate::kernel!(
813 #[inline(always)]
814 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> i32x4<Sse4_2> {
815 _mm_cvttps_epi32(a.into()).simd_into(token)
816 }
817 );
818 kernel(self, a)
819 }
820 #[inline(always)]
821 fn cvt_i32_precise_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
822 crate::kernel!(
823 #[inline(always)]
824 fn kernel(token: Sse4_2, a: f32x4<Sse4_2>) -> i32x4<Sse4_2> {
825 let a = a.into();
826 let converted = _mm_cvttps_epi32(a);
827 let positive_overflow =
828 _mm_castps_si128(_mm_cmple_ps(_mm_set1_ps(2147483648.0), a));
829 let converted = _mm_xor_si128(converted, positive_overflow);
830 let is_not_nan = _mm_castps_si128(_mm_cmpord_ps(a, a));
831 _mm_and_si128(converted, is_not_nan).simd_into(token)
832 }
833 );
834 kernel(self, a)
835 }
836 #[inline(always)]
837 fn abs_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
838 crate::kernel!(
839 #[inline(always)]
840 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
841 _mm_abs_epi8(a.into()).simd_into(token)
842 }
843 );
844 kernel(self, a)
845 }
846 #[inline(always)]
847 fn splat_i8x16(self, val: i8) -> i8x16<Self> {
848 crate::kernel!(
849 #[inline(always)]
850 fn kernel(token: Sse4_2, val: i8) -> i8x16<Sse4_2> {
851 _mm_set1_epi8(val).simd_into(token)
852 }
853 );
854 kernel(self, val)
855 }
856 #[inline(always)]
857 fn reverse_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
858 let indices: u8x16<Self> =
859 [15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0].simd_into(self);
860 self.swizzle_dyn_i8x16(a, indices)
861 }
862 #[inline(always)]
863 fn slide_i8x16<const SHIFT: usize>(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
864 if SHIFT >= 16usize {
865 return b;
866 }
867 let result = dyn_alignr_128(
868 self,
869 Bytes::to_bytes(b).val.0,
870 Bytes::to_bytes(a).val.0,
871 SHIFT,
872 );
873 Bytes::from_bytes(u8x16 {
874 val: crate::support::Aligned128(result),
875 simd: self,
876 })
877 }
878 #[inline(always)]
879 fn compress_i8x16(self, values: i8x16<Self>, mask: mask8x16<Self>) -> i8x16<Self> {
880 let mask = mask8x16 {
881 val: crate::transmute::checked_transmute_copy(&mask.val),
882 simd: self,
883 };
884 Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask))
885 }
886 #[inline(always)]
887 fn compress_merge_i8x16(
888 self,
889 values: i8x16<Self>,
890 mask: mask8x16<Self>,
891 merge: i8x16<Self>,
892 ) -> i8x16<Self> {
893 let mask = mask8x16 {
894 val: crate::transmute::checked_transmute_copy(&mask.val),
895 simd: self,
896 };
897 Bytes::from_bytes(self.compress_merge_u8x16(
898 Bytes::to_bytes(values),
899 mask,
900 Bytes::to_bytes(merge),
901 ))
902 }
903 #[inline(always)]
904 fn expand_i8x16(self, values: i8x16<Self>, mask: mask8x16<Self>) -> i8x16<Self> {
905 let mask = mask8x16 {
906 val: crate::transmute::checked_transmute_copy(&mask.val),
907 simd: self,
908 };
909 Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask))
910 }
911 #[inline(always)]
912 fn expand_merge_i8x16(
913 self,
914 values: i8x16<Self>,
915 mask: mask8x16<Self>,
916 merge: i8x16<Self>,
917 ) -> i8x16<Self> {
918 let mask = mask8x16 {
919 val: crate::transmute::checked_transmute_copy(&mask.val),
920 simd: self,
921 };
922 Bytes::from_bytes(self.expand_merge_u8x16(
923 Bytes::to_bytes(values),
924 mask,
925 Bytes::to_bytes(merge),
926 ))
927 }
928 #[inline(always)]
929 fn count_ones_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
930 crate::kernel!(
931 #[inline(always)]
932 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
933 let value = a.into();
934 let nibble_mask = _mm_set1_epi8(0x0f);
935 let lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4);
936 let low = _mm_and_si128(value, nibble_mask);
937 let high = _mm_and_si128(_mm_srli_epi16::<4>(value), nibble_mask);
938 let byte_counts = _mm_add_epi8(
939 _mm_shuffle_epi8(lookup, low),
940 _mm_shuffle_epi8(lookup, high),
941 );
942 byte_counts.simd_into(token)
943 }
944 );
945 kernel(self, a)
946 }
947 #[inline(always)]
948 fn count_zeros_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
949 self.count_ones_i8x16(self.not_i8x16(a))
950 }
951 #[inline(always)]
952 fn add_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
953 crate::kernel!(
954 #[inline(always)]
955 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
956 _mm_add_epi8(a.into(), b.into()).simd_into(token)
957 }
958 );
959 kernel(self, a, b)
960 }
961 #[inline(always)]
962 fn saturating_add_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
963 crate::kernel!(
964 #[inline(always)]
965 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
966 _mm_adds_epi8(a.into(), b.into()).simd_into(token)
967 }
968 );
969 kernel(self, a, b)
970 }
971 #[inline(always)]
972 fn sub_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
973 crate::kernel!(
974 #[inline(always)]
975 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
976 _mm_sub_epi8(a.into(), b.into()).simd_into(token)
977 }
978 );
979 kernel(self, a, b)
980 }
981 #[inline(always)]
982 fn saturating_sub_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
983 crate::kernel!(
984 #[inline(always)]
985 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
986 _mm_subs_epi8(a.into(), b.into()).simd_into(token)
987 }
988 );
989 kernel(self, a, b)
990 }
991 #[inline(always)]
992 fn mul_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
993 crate::kernel!(
994 #[inline(always)]
995 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
996 let a = a.into();
997 let b = b.into();
998 let low_mask = _mm_set1_epi16(0xFF);
999 let dst_even = _mm_mullo_epi16(a, b);
1000 let dst_odd = _mm_maddubs_epi16(a, _mm_andnot_si128(low_mask, b));
1001 _mm_or_si128(
1002 _mm_slli_epi16(dst_odd, 8),
1003 _mm_and_si128(dst_even, low_mask),
1004 )
1005 .simd_into(token)
1006 }
1007 );
1008 kernel(self, a, b)
1009 }
1010 #[inline(always)]
1011 fn and_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1012 crate::kernel!(
1013 #[inline(always)]
1014 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
1015 _mm_and_si128(a.into(), b.into()).simd_into(token)
1016 }
1017 );
1018 kernel(self, a, b)
1019 }
1020 #[inline(always)]
1021 fn or_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1022 crate::kernel!(
1023 #[inline(always)]
1024 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
1025 _mm_or_si128(a.into(), b.into()).simd_into(token)
1026 }
1027 );
1028 kernel(self, a, b)
1029 }
1030 #[inline(always)]
1031 fn xor_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1032 crate::kernel!(
1033 #[inline(always)]
1034 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
1035 _mm_xor_si128(a.into(), b.into()).simd_into(token)
1036 }
1037 );
1038 kernel(self, a, b)
1039 }
1040 #[inline(always)]
1041 fn not_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
1042 a ^ !0
1043 }
1044 #[inline(always)]
1045 fn shl_i8x16(self, a: i8x16<Self>, shift: u32) -> i8x16<Self> {
1046 crate::kernel!(
1047 #[inline(always)]
1048 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, shift: u32) -> i8x16<Sse4_2> {
1049 let val = a.into();
1050 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
1051 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
1052 let byte_mask = _mm_set1_epi8(mask_byte);
1053 _mm_sll_epi16(_mm_and_si128(val, byte_mask), shift_count).simd_into(token)
1054 }
1055 );
1056 kernel(self, a, shift)
1057 }
1058 #[inline(always)]
1059 fn shlv_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1060 [
1061 i8::wrapping_shl(a[0usize], b[0usize] as u32),
1062 i8::wrapping_shl(a[1usize], b[1usize] as u32),
1063 i8::wrapping_shl(a[2usize], b[2usize] as u32),
1064 i8::wrapping_shl(a[3usize], b[3usize] as u32),
1065 i8::wrapping_shl(a[4usize], b[4usize] as u32),
1066 i8::wrapping_shl(a[5usize], b[5usize] as u32),
1067 i8::wrapping_shl(a[6usize], b[6usize] as u32),
1068 i8::wrapping_shl(a[7usize], b[7usize] as u32),
1069 i8::wrapping_shl(a[8usize], b[8usize] as u32),
1070 i8::wrapping_shl(a[9usize], b[9usize] as u32),
1071 i8::wrapping_shl(a[10usize], b[10usize] as u32),
1072 i8::wrapping_shl(a[11usize], b[11usize] as u32),
1073 i8::wrapping_shl(a[12usize], b[12usize] as u32),
1074 i8::wrapping_shl(a[13usize], b[13usize] as u32),
1075 i8::wrapping_shl(a[14usize], b[14usize] as u32),
1076 i8::wrapping_shl(a[15usize], b[15usize] as u32),
1077 ]
1078 .simd_into(self)
1079 }
1080 #[inline(always)]
1081 fn shr_i8x16(self, a: i8x16<Self>, shift: u32) -> i8x16<Self> {
1082 crate::kernel!(
1083 #[inline(always)]
1084 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, shift: u32) -> i8x16<Sse4_2> {
1085 let val = a.into();
1086 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
1087 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
1088 let byte_mask = _mm_set1_epi8(mask_byte);
1089 let shifted = _mm_srl_epi16(val, shift_count);
1090 let result = {
1091 let sign = _mm_cmpgt_epi8(_mm_setzero_si128(), val);
1092 _mm_or_si128(
1093 _mm_and_si128(shifted, byte_mask),
1094 _mm_andnot_si128(byte_mask, sign),
1095 )
1096 };
1097 result.simd_into(token)
1098 }
1099 );
1100 kernel(self, a, shift)
1101 }
1102 #[inline(always)]
1103 fn shrv_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1104 [
1105 i8::wrapping_shr(a[0usize], b[0usize] as u32),
1106 i8::wrapping_shr(a[1usize], b[1usize] as u32),
1107 i8::wrapping_shr(a[2usize], b[2usize] as u32),
1108 i8::wrapping_shr(a[3usize], b[3usize] as u32),
1109 i8::wrapping_shr(a[4usize], b[4usize] as u32),
1110 i8::wrapping_shr(a[5usize], b[5usize] as u32),
1111 i8::wrapping_shr(a[6usize], b[6usize] as u32),
1112 i8::wrapping_shr(a[7usize], b[7usize] as u32),
1113 i8::wrapping_shr(a[8usize], b[8usize] as u32),
1114 i8::wrapping_shr(a[9usize], b[9usize] as u32),
1115 i8::wrapping_shr(a[10usize], b[10usize] as u32),
1116 i8::wrapping_shr(a[11usize], b[11usize] as u32),
1117 i8::wrapping_shr(a[12usize], b[12usize] as u32),
1118 i8::wrapping_shr(a[13usize], b[13usize] as u32),
1119 i8::wrapping_shr(a[14usize], b[14usize] as u32),
1120 i8::wrapping_shr(a[15usize], b[15usize] as u32),
1121 ]
1122 .simd_into(self)
1123 }
1124 #[inline(always)]
1125 fn reduce_max_i8x16(self, a: i8x16<Self>) -> i8 {
1126 crate::kernel!(
1127 #[inline(always)]
1128 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>) -> i8 {
1129 let reduced: __m128i = a.into();
1130 let shifted = _mm_srli_si128::<8>(reduced);
1131 let reduced = { _mm_max_epi8(reduced, shifted) };
1132 let shifted = _mm_srli_si128::<4>(reduced);
1133 let reduced = { _mm_max_epi8(reduced, shifted) };
1134 let shifted = _mm_srli_si128::<2>(reduced);
1135 let reduced = { _mm_max_epi8(reduced, shifted) };
1136 let shifted = _mm_srli_si128::<1>(reduced);
1137 let reduced = { _mm_max_epi8(reduced, shifted) };
1138 {
1139 let lanes: [i8; 16usize] = crate::transmute::checked_transmute_copy(&reduced);
1140 lanes[0]
1141 }
1142 }
1143 );
1144 kernel(self, a)
1145 }
1146 #[inline(always)]
1147 fn reduce_min_i8x16(self, a: i8x16<Self>) -> i8 {
1148 crate::kernel!(
1149 #[inline(always)]
1150 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>) -> i8 {
1151 let reduced: __m128i = a.into();
1152 let shifted = _mm_srli_si128::<8>(reduced);
1153 let reduced = { _mm_min_epi8(reduced, shifted) };
1154 let shifted = _mm_srli_si128::<4>(reduced);
1155 let reduced = { _mm_min_epi8(reduced, shifted) };
1156 let shifted = _mm_srli_si128::<2>(reduced);
1157 let reduced = { _mm_min_epi8(reduced, shifted) };
1158 let shifted = _mm_srli_si128::<1>(reduced);
1159 let reduced = { _mm_min_epi8(reduced, shifted) };
1160 {
1161 let lanes: [i8; 16usize] = crate::transmute::checked_transmute_copy(&reduced);
1162 lanes[0]
1163 }
1164 }
1165 );
1166 kernel(self, a)
1167 }
1168 #[inline(always)]
1169 fn reduce_sum_i8x16(self, a: i8x16<Self>) -> i8 {
1170 crate::kernel!(
1171 #[inline(always)]
1172 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>) -> i8 {
1173 let sum: __m128i = a.into();
1174 let sum = _mm_add_epi8(sum, _mm_srli_si128::<8>(sum));
1175 let sum = _mm_add_epi8(sum, _mm_srli_si128::<4>(sum));
1176 let sum = _mm_add_epi8(sum, _mm_srli_si128::<2>(sum));
1177 let sum = _mm_add_epi8(sum, _mm_srli_si128::<1>(sum));
1178 let lanes: [i8; 16usize] = crate::transmute::checked_transmute_copy(&sum);
1179 lanes[0]
1180 }
1181 );
1182 kernel(self, a)
1183 }
1184 #[inline(always)]
1185 fn reduce_product_i8x16(self, a: i8x16<Self>) -> i8 {
1186 crate::kernel!(
1187 #[inline(always)]
1188 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>) -> i8 {
1189 let value: __m128i = a.into();
1190 let product = _mm_mullo_epi16(
1191 _mm_unpacklo_epi8(value, value),
1192 _mm_unpackhi_epi8(value, value),
1193 );
1194 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b11_10_11_10>(product));
1195 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b01_01_01_01>(product));
1196 let product = _mm_mullo_epi16(product, _mm_srli_epi32::<16>(product));
1197 let lanes: [i8; 16usize] = crate::transmute::checked_transmute_copy(&product);
1198 lanes[0]
1199 }
1200 );
1201 kernel(self, a)
1202 }
1203 #[inline(always)]
1204 fn max_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1205 crate::kernel!(
1206 #[inline(always)]
1207 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
1208 _mm_max_epi8(a.into(), b.into()).simd_into(token)
1209 }
1210 );
1211 kernel(self, a, b)
1212 }
1213 #[inline(always)]
1214 fn min_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1215 crate::kernel!(
1216 #[inline(always)]
1217 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
1218 _mm_min_epi8(a.into(), b.into()).simd_into(token)
1219 }
1220 );
1221 kernel(self, a, b)
1222 }
1223 #[inline(always)]
1224 fn clamp_i8x16(self, a: i8x16<Self>, b: i8x16<Self>, c: i8x16<Self>) -> i8x16<Self> {
1225 debug_assert!(
1226 b.simd_le(c).all_true(),
1227 "clamp requires min <= max for all elements"
1228 );
1229 a.max(b).min(c)
1230 }
1231 #[inline(always)]
1232 fn simd_eq_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
1233 crate::kernel!(
1234 #[inline(always)]
1235 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> mask8x16<Sse4_2> {
1236 _mm_cmpeq_epi8(a.into(), b.into()).simd_into(token)
1237 }
1238 );
1239 kernel(self, a, b)
1240 }
1241 #[inline(always)]
1242 fn simd_lt_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
1243 crate::kernel!(
1244 #[inline(always)]
1245 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> mask8x16<Sse4_2> {
1246 _mm_cmpgt_epi8(b.into(), a.into()).simd_into(token)
1247 }
1248 );
1249 kernel(self, a, b)
1250 }
1251 #[inline(always)]
1252 fn simd_le_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
1253 crate::kernel!(
1254 #[inline(always)]
1255 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> mask8x16<Sse4_2> {
1256 _mm_cmpeq_epi8(_mm_min_epi8(a.into(), b.into()), a.into()).simd_into(token)
1257 }
1258 );
1259 kernel(self, a, b)
1260 }
1261 #[inline(always)]
1262 fn zip_low_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1263 crate::kernel!(
1264 #[inline(always)]
1265 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
1266 _mm_unpacklo_epi8(a.into(), b.into()).simd_into(token)
1267 }
1268 );
1269 kernel(self, a, b)
1270 }
1271 #[inline(always)]
1272 fn zip_high_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1273 crate::kernel!(
1274 #[inline(always)]
1275 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
1276 _mm_unpackhi_epi8(a.into(), b.into()).simd_into(token)
1277 }
1278 );
1279 kernel(self, a, b)
1280 }
1281 #[inline(always)]
1282 fn unzip_low_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1283 crate::kernel!(
1284 #[inline(always)]
1285 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
1286 let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
1287 let t1 = _mm_shuffle_epi8(a.into(), mask);
1288 let t2 = _mm_shuffle_epi8(b.into(), mask);
1289 _mm_unpacklo_epi64(t1, t2).simd_into(token)
1290 }
1291 );
1292 kernel(self, a, b)
1293 }
1294 #[inline(always)]
1295 fn unzip_high_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1296 crate::kernel!(
1297 #[inline(always)]
1298 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>, b: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
1299 let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
1300 let t1 = _mm_shuffle_epi8(a.into(), mask);
1301 let t2 = _mm_shuffle_epi8(b.into(), mask);
1302 _mm_unpackhi_epi64(t1, t2).simd_into(token)
1303 }
1304 );
1305 kernel(self, a, b)
1306 }
1307 #[inline(always)]
1308 fn interleave_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i8x16<Self>, i8x16<Self>) {
1309 (self.zip_low_i8x16(a, b), self.zip_high_i8x16(a, b))
1310 }
1311 #[inline(always)]
1312 fn deinterleave_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i8x16<Self>, i8x16<Self>) {
1313 crate::kernel!(
1314 #[inline(always)]
1315 fn kernel(
1316 token: Sse4_2,
1317 a: i8x16<Sse4_2>,
1318 b: i8x16<Sse4_2>,
1319 ) -> (i8x16<Sse4_2>, i8x16<Sse4_2>) {
1320 let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
1321 let a = _mm_shuffle_epi8(a.into(), mask);
1322 let b = _mm_shuffle_epi8(b.into(), mask);
1323 (
1324 _mm_unpacklo_epi64(a, b).simd_into(token),
1325 _mm_unpackhi_epi64(a, b).simd_into(token),
1326 )
1327 }
1328 );
1329 kernel(self, a, b)
1330 }
1331 #[inline(always)]
1332 fn select_i8x16(self, a: mask8x16<Self>, b: i8x16<Self>, c: i8x16<Self>) -> i8x16<Self> {
1333 crate::kernel!(
1334 #[inline(always)]
1335 fn kernel(
1336 token: Sse4_2,
1337 a: mask8x16<Sse4_2>,
1338 b: i8x16<Sse4_2>,
1339 c: i8x16<Sse4_2>,
1340 ) -> i8x16<Sse4_2> {
1341 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
1342 }
1343 );
1344 kernel(self, a, b, c)
1345 }
1346 #[inline(always)]
1347 fn combine_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x32<Self> {
1348 i8x32 {
1349 val: crate::support::Aligned256([a.val.0, b.val.0]),
1350 simd: self,
1351 }
1352 }
1353 #[inline(always)]
1354 fn neg_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
1355 crate::kernel!(
1356 #[inline(always)]
1357 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>) -> i8x16<Sse4_2> {
1358 _mm_sub_epi8(_mm_setzero_si128(), a.into()).simd_into(token)
1359 }
1360 );
1361 kernel(self, a)
1362 }
1363 #[inline(always)]
1364 fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16<Self>; 4usize] {
1365 crate::kernel!(
1366 #[inline(always)]
1367 fn kernel(token: Sse4_2, src: &[i8; 64usize]) -> [i8x16<Sse4_2>; 4usize] {
1368 let (chunks, []) = src.as_chunks::<16usize>() else {
1369 unreachable!()
1370 };
1371 let v0: __m128i =
1372 crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[0]);
1373 let v1: __m128i =
1374 crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[1]);
1375 let v2: __m128i =
1376 crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[2]);
1377 let v3: __m128i =
1378 crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[3]);
1379 let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
1380 let v0 = _mm_shuffle_epi8(v0, mask);
1381 let v1 = _mm_shuffle_epi8(v1, mask);
1382 let v2 = _mm_shuffle_epi8(v2, mask);
1383 let v3 = _mm_shuffle_epi8(v3, mask);
1384 let tmp0 = _mm_unpacklo_epi32(v0, v1);
1385 let tmp1 = _mm_unpackhi_epi32(v0, v1);
1386 let tmp2 = _mm_unpacklo_epi32(v2, v3);
1387 let tmp3 = _mm_unpackhi_epi32(v2, v3);
1388 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
1389 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
1390 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
1391 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
1392 [
1393 out0.simd_into(token),
1394 out1.simd_into(token),
1395 out2.simd_into(token),
1396 out3.simd_into(token),
1397 ]
1398 }
1399 );
1400 kernel(self, src)
1401 }
1402 #[inline(always)]
1403 fn store_four_interleaved_i8x16(
1404 self,
1405 vectors: [i8x16<Self>; 4usize],
1406 dest: &mut [i8; 64usize],
1407 ) -> () {
1408 crate::kernel!(
1409 #[inline(always)]
1410 fn kernel(
1411 token: Sse4_2,
1412 vectors: [i8x16<Sse4_2>; 4usize],
1413 dest: &mut [i8; 64usize],
1414 ) -> () {
1415 let _ = token;
1416 let v0: __m128i = vectors[0].into();
1417 let v1: __m128i = vectors[1].into();
1418 let v2: __m128i = vectors[2].into();
1419 let v3: __m128i = vectors[3].into();
1420 let tmp0 = _mm_unpacklo_epi32(v0, v1);
1421 let tmp1 = _mm_unpackhi_epi32(v0, v1);
1422 let tmp2 = _mm_unpacklo_epi32(v2, v3);
1423 let tmp3 = _mm_unpackhi_epi32(v2, v3);
1424 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
1425 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
1426 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
1427 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
1428 let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
1429 let out0 = _mm_shuffle_epi8(out0, mask);
1430 let out1 = _mm_shuffle_epi8(out1, mask);
1431 let out2 = _mm_shuffle_epi8(out2, mask);
1432 let out3 = _mm_shuffle_epi8(out3, mask);
1433 let (chunks, []) = dest.as_chunks_mut::<16usize>() else {
1434 unreachable!()
1435 };
1436 crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>(
1437 out0,
1438 &mut chunks[0],
1439 );
1440 crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>(
1441 out1,
1442 &mut chunks[1],
1443 );
1444 crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>(
1445 out2,
1446 &mut chunks[2],
1447 );
1448 crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>(
1449 out3,
1450 &mut chunks[3],
1451 );
1452 }
1453 );
1454 kernel(self, vectors, dest);
1455 }
1456 #[inline(always)]
1457 fn widen_i8x16(self, a: i8x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
1458 crate::kernel!(
1459 #[inline(always)]
1460 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>) -> (i16x8<Sse4_2>, i16x8<Sse4_2>) {
1461 let raw = a.into();
1462 (
1463 _mm_cvtepi8_epi16(raw).simd_into(token),
1464 _mm_cvtepi8_epi16(_mm_srli_si128::<8>(raw)).simd_into(token),
1465 )
1466 }
1467 );
1468 kernel(self, a)
1469 }
1470 #[inline(always)]
1471 fn widening_mul_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
1472 let (a0, a1) = self.widen_i8x16(a);
1473 let (b0, b1) = self.widen_i8x16(b);
1474 (self.mul_i16x8(a0, b0), self.mul_i16x8(a1, b1))
1475 }
1476 #[inline(always)]
1477 fn widening_add_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
1478 let (a0, a1) = self.widen_i8x16(a);
1479 let (b0, b1) = self.widen_i8x16(b);
1480 (self.add_i16x8(a0, b0), self.add_i16x8(a1, b1))
1481 }
1482 #[inline(always)]
1483 fn pairwise_widening_add_i8x16(self, a: i8x16<Self>) -> i16x8<Self> {
1484 crate::kernel!(
1485 #[inline(always)]
1486 fn kernel(token: Sse4_2, a: i8x16<Sse4_2>) -> i16x8<Sse4_2> {
1487 ({ _mm_maddubs_epi16(_mm_set1_epi8(1), a.into()) }).simd_into(token)
1488 }
1489 );
1490 kernel(self, a)
1491 }
1492 #[inline(always)]
1493 fn pairwise_dot_product_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i16x8<Self> {
1494 let (low, high) = self.widening_mul_i8x16(a, b);
1495 let (even, odd) = self.deinterleave_i16x8(low, high);
1496 self.add_i16x8(even, odd)
1497 }
1498 #[inline(always)]
1499 fn splat_u8x16(self, val: u8) -> u8x16<Self> {
1500 crate::kernel!(
1501 #[inline(always)]
1502 fn kernel(token: Sse4_2, val: u8) -> u8x16<Sse4_2> {
1503 _mm_set1_epi8(val.cast_signed()).simd_into(token)
1504 }
1505 );
1506 kernel(self, val)
1507 }
1508 #[inline(always)]
1509 fn reverse_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
1510 let indices: u8x16<Self> =
1511 [15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0].simd_into(self);
1512 self.swizzle_dyn_u8x16(a, indices)
1513 }
1514 #[inline(always)]
1515 fn slide_u8x16<const SHIFT: usize>(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1516 if SHIFT >= 16usize {
1517 return b;
1518 }
1519 let result = dyn_alignr_128(
1520 self,
1521 Bytes::to_bytes(b).val.0,
1522 Bytes::to_bytes(a).val.0,
1523 SHIFT,
1524 );
1525 Bytes::from_bytes(u8x16 {
1526 val: crate::support::Aligned128(result),
1527 simd: self,
1528 })
1529 }
1530 #[inline(always)]
1531 fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
1532 crate::kernel!(
1533 #[inline(always)]
1534 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, indices: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1535 let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, indices.into());
1536 Bytes::from_bytes(u8x16 {
1537 val: crate::support::Aligned128(result),
1538 simd: token,
1539 })
1540 }
1541 );
1542 kernel(self, a, indices)
1543 }
1544 #[inline(always)]
1545 fn swizzle_dyn_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
1546 crate::kernel!(
1547 #[inline(always)]
1548 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, indices: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1549 let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, indices.into());
1550 Bytes::from_bytes(u8x16 {
1551 val: crate::support::Aligned128(result),
1552 simd: token,
1553 })
1554 }
1555 );
1556 kernel(self, a, indices)
1557 }
1558 #[inline(always)]
1559 fn swizzle_dyn_precise_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
1560 crate::kernel!(
1561 #[inline(always)]
1562 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, indices: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1563 let indices = indices.into();
1564 let zeroing_indices = _mm_adds_epu8(indices, _mm_set1_epi8(0x70));
1565 let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices);
1566 let result_bytes = u8x16 {
1567 val: crate::support::Aligned128(result),
1568 simd: token,
1569 };
1570 Bytes::from_bytes(result_bytes)
1571 }
1572 );
1573 kernel(self, a, indices)
1574 }
1575 #[inline(always)]
1576 fn concat_swizzle_dyn_u8x16(
1577 self,
1578 a: u8x16<Self>,
1579 b: u8x16<Self>,
1580 indices: u8x16<Self>,
1581 ) -> u8x16<Self> {
1582 self.concat_swizzle_dyn_precise_u8x16(a, b, indices)
1583 }
1584 #[inline(always)]
1585 fn concat_swizzle_dyn_precise_u8x16(
1586 self,
1587 a: u8x16<Self>,
1588 b: u8x16<Self>,
1589 indices: u8x16<Self>,
1590 ) -> u8x16<Self> {
1591 let first_table = Bytes::to_bytes(a);
1592 let second_table = Bytes::to_bytes(b);
1593 let second_table_offset = self.splat_u8x16(16);
1594 let from_first = self.swizzle_dyn_precise_u8x16(first_table, indices);
1595 let from_second = self
1596 .swizzle_dyn_precise_u8x16(second_table, self.sub_u8x16(indices, second_table_offset));
1597 let result_bytes = self.or_u8x16(from_first, from_second);
1598 Bytes::from_bytes(result_bytes)
1599 }
1600 #[inline(always)]
1601 fn compress_u8x16(self, values: u8x16<Self>, mask: mask8x16<Self>) -> u8x16<Self> {
1602 crate::kernel!(
1603 #[inline(always)]
1604 fn kernel(
1605 token: Sse4_2,
1606 values: u8x16<Sse4_2>,
1607 mask: mask8x16<Sse4_2>,
1608 ) -> u8x16<Sse4_2> {
1609 let mask_bits = token.to_bitmask_mask8x16(mask) as u16;
1610 let low_mask = (usize::from(mask_bits)) & 0xff;
1611 let high_mask = (usize::from(mask_bits)) >> 8;
1612 let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask];
1613 let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask];
1614 let high_control = high_control | 0x0808_0808_0808_0808;
1615 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
1616 let low_count = low_mask.count_ones() as usize;
1617 let compacted = _mm_shuffle_epi8(values.into(), control);
1618 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count];
1619 let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
1620 let compressed = _mm_shuffle_epi8(compacted, splice);
1621 compressed.simd_into(token)
1622 }
1623 );
1624 kernel(self, values, mask)
1625 }
1626 #[inline(always)]
1627 fn compress_merge_u8x16(
1628 self,
1629 values: u8x16<Self>,
1630 mask: mask8x16<Self>,
1631 merge: u8x16<Self>,
1632 ) -> u8x16<Self> {
1633 crate::kernel!(
1634 #[inline(always)]
1635 fn kernel(
1636 token: Sse4_2,
1637 values: u8x16<Sse4_2>,
1638 mask: mask8x16<Sse4_2>,
1639 merge: u8x16<Sse4_2>,
1640 ) -> u8x16<Sse4_2> {
1641 let mask_bits = token.to_bitmask_mask8x16(mask) as u16;
1642 let low_mask = (usize::from(mask_bits)) & 0xff;
1643 let high_mask = (usize::from(mask_bits)) >> 8;
1644 let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask];
1645 let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask];
1646 let high_control = high_control | 0x0808_0808_0808_0808;
1647 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
1648 let low_count = low_mask.count_ones() as usize;
1649 let compacted = _mm_shuffle_epi8(values.into(), control);
1650 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count];
1651 let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
1652 let compressed = _mm_shuffle_epi8(compacted, splice);
1653 let count = mask_bits.count_ones() as usize;
1654 let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>(
1655 &crate::support::COMPACT_PREFIX_MASKS[count],
1656 );
1657 _mm_blendv_epi8(merge.into(), compressed, prefix).simd_into(token)
1658 }
1659 );
1660 kernel(self, values, mask, merge)
1661 }
1662 #[inline(always)]
1663 fn expand_u8x16(self, values: u8x16<Self>, mask: mask8x16<Self>) -> u8x16<Self> {
1664 crate::kernel!(
1665 #[inline(always)]
1666 fn kernel(
1667 token: Sse4_2,
1668 values: u8x16<Sse4_2>,
1669 mask: mask8x16<Sse4_2>,
1670 ) -> u8x16<Sse4_2> {
1671 let mask_bits = token.to_bitmask_mask8x16(mask) as u16;
1672 let low_mask = (usize::from(mask_bits)) & 0xff;
1673 let high_mask = (usize::from(mask_bits)) >> 8;
1674 let low_count = low_mask.count_ones() as u64;
1675 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
1676 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
1677 let high_base = low_count * 0x0101_0101_0101_0101;
1678 let high_control = high_control + high_base;
1679 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
1680 let expanded = _mm_shuffle_epi8(values.into(), control);
1681 expanded.simd_into(token)
1682 }
1683 );
1684 kernel(self, values, mask)
1685 }
1686 #[inline(always)]
1687 fn expand_merge_u8x16(
1688 self,
1689 values: u8x16<Self>,
1690 mask: mask8x16<Self>,
1691 merge: u8x16<Self>,
1692 ) -> u8x16<Self> {
1693 crate::kernel!(
1694 #[inline(always)]
1695 fn kernel(
1696 token: Sse4_2,
1697 values: u8x16<Sse4_2>,
1698 mask: mask8x16<Sse4_2>,
1699 merge: u8x16<Sse4_2>,
1700 ) -> u8x16<Sse4_2> {
1701 let mask_bits = token.to_bitmask_mask8x16(mask) as u16;
1702 let low_mask = (usize::from(mask_bits)) & 0xff;
1703 let high_mask = (usize::from(mask_bits)) >> 8;
1704 let low_count = low_mask.count_ones() as u64;
1705 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
1706 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
1707 let high_base = low_count * 0x0101_0101_0101_0101;
1708 let high_control = high_control + high_base;
1709 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
1710 let expanded = _mm_shuffle_epi8(values.into(), control);
1711 _mm_blendv_epi8(merge.into(), expanded, mask.into()).simd_into(token)
1712 }
1713 );
1714 kernel(self, values, mask, merge)
1715 }
1716 #[inline(always)]
1717 fn count_ones_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
1718 crate::kernel!(
1719 #[inline(always)]
1720 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1721 let value = a.into();
1722 let nibble_mask = _mm_set1_epi8(0x0f);
1723 let lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4);
1724 let low = _mm_and_si128(value, nibble_mask);
1725 let high = _mm_and_si128(_mm_srli_epi16::<4>(value), nibble_mask);
1726 let byte_counts = _mm_add_epi8(
1727 _mm_shuffle_epi8(lookup, low),
1728 _mm_shuffle_epi8(lookup, high),
1729 );
1730 byte_counts.simd_into(token)
1731 }
1732 );
1733 kernel(self, a)
1734 }
1735 #[inline(always)]
1736 fn count_zeros_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
1737 self.count_ones_u8x16(self.not_u8x16(a))
1738 }
1739 #[inline(always)]
1740 fn add_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1741 crate::kernel!(
1742 #[inline(always)]
1743 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1744 _mm_add_epi8(a.into(), b.into()).simd_into(token)
1745 }
1746 );
1747 kernel(self, a, b)
1748 }
1749 #[inline(always)]
1750 fn saturating_add_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1751 crate::kernel!(
1752 #[inline(always)]
1753 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1754 _mm_adds_epu8(a.into(), b.into()).simd_into(token)
1755 }
1756 );
1757 kernel(self, a, b)
1758 }
1759 #[inline(always)]
1760 fn sub_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1761 crate::kernel!(
1762 #[inline(always)]
1763 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1764 _mm_sub_epi8(a.into(), b.into()).simd_into(token)
1765 }
1766 );
1767 kernel(self, a, b)
1768 }
1769 #[inline(always)]
1770 fn saturating_sub_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1771 crate::kernel!(
1772 #[inline(always)]
1773 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1774 _mm_subs_epu8(a.into(), b.into()).simd_into(token)
1775 }
1776 );
1777 kernel(self, a, b)
1778 }
1779 #[inline(always)]
1780 fn mul_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1781 crate::kernel!(
1782 #[inline(always)]
1783 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1784 let a = a.into();
1785 let b = b.into();
1786 let low_mask = _mm_set1_epi16(0xFF);
1787 let dst_even = _mm_mullo_epi16(a, b);
1788 let dst_odd = _mm_maddubs_epi16(a, _mm_andnot_si128(low_mask, b));
1789 _mm_or_si128(
1790 _mm_slli_epi16(dst_odd, 8),
1791 _mm_and_si128(dst_even, low_mask),
1792 )
1793 .simd_into(token)
1794 }
1795 );
1796 kernel(self, a, b)
1797 }
1798 #[inline(always)]
1799 fn and_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1800 crate::kernel!(
1801 #[inline(always)]
1802 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1803 _mm_and_si128(a.into(), b.into()).simd_into(token)
1804 }
1805 );
1806 kernel(self, a, b)
1807 }
1808 #[inline(always)]
1809 fn or_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1810 crate::kernel!(
1811 #[inline(always)]
1812 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1813 _mm_or_si128(a.into(), b.into()).simd_into(token)
1814 }
1815 );
1816 kernel(self, a, b)
1817 }
1818 #[inline(always)]
1819 fn xor_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1820 crate::kernel!(
1821 #[inline(always)]
1822 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1823 _mm_xor_si128(a.into(), b.into()).simd_into(token)
1824 }
1825 );
1826 kernel(self, a, b)
1827 }
1828 #[inline(always)]
1829 fn not_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
1830 a ^ !0
1831 }
1832 #[inline(always)]
1833 fn shl_u8x16(self, a: u8x16<Self>, shift: u32) -> u8x16<Self> {
1834 crate::kernel!(
1835 #[inline(always)]
1836 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, shift: u32) -> u8x16<Sse4_2> {
1837 let val = a.into();
1838 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
1839 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
1840 let byte_mask = _mm_set1_epi8(mask_byte);
1841 _mm_sll_epi16(_mm_and_si128(val, byte_mask), shift_count).simd_into(token)
1842 }
1843 );
1844 kernel(self, a, shift)
1845 }
1846 #[inline(always)]
1847 fn shlv_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1848 [
1849 u8::wrapping_shl(a[0usize], b[0usize] as u32),
1850 u8::wrapping_shl(a[1usize], b[1usize] as u32),
1851 u8::wrapping_shl(a[2usize], b[2usize] as u32),
1852 u8::wrapping_shl(a[3usize], b[3usize] as u32),
1853 u8::wrapping_shl(a[4usize], b[4usize] as u32),
1854 u8::wrapping_shl(a[5usize], b[5usize] as u32),
1855 u8::wrapping_shl(a[6usize], b[6usize] as u32),
1856 u8::wrapping_shl(a[7usize], b[7usize] as u32),
1857 u8::wrapping_shl(a[8usize], b[8usize] as u32),
1858 u8::wrapping_shl(a[9usize], b[9usize] as u32),
1859 u8::wrapping_shl(a[10usize], b[10usize] as u32),
1860 u8::wrapping_shl(a[11usize], b[11usize] as u32),
1861 u8::wrapping_shl(a[12usize], b[12usize] as u32),
1862 u8::wrapping_shl(a[13usize], b[13usize] as u32),
1863 u8::wrapping_shl(a[14usize], b[14usize] as u32),
1864 u8::wrapping_shl(a[15usize], b[15usize] as u32),
1865 ]
1866 .simd_into(self)
1867 }
1868 #[inline(always)]
1869 fn shr_u8x16(self, a: u8x16<Self>, shift: u32) -> u8x16<Self> {
1870 crate::kernel!(
1871 #[inline(always)]
1872 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, shift: u32) -> u8x16<Sse4_2> {
1873 let val = a.into();
1874 let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
1875 let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
1876 let byte_mask = _mm_set1_epi8(mask_byte);
1877 let shifted = _mm_srl_epi16(val, shift_count);
1878 let result = { _mm_and_si128(shifted, byte_mask) };
1879 result.simd_into(token)
1880 }
1881 );
1882 kernel(self, a, shift)
1883 }
1884 #[inline(always)]
1885 fn shrv_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1886 [
1887 u8::wrapping_shr(a[0usize], b[0usize] as u32),
1888 u8::wrapping_shr(a[1usize], b[1usize] as u32),
1889 u8::wrapping_shr(a[2usize], b[2usize] as u32),
1890 u8::wrapping_shr(a[3usize], b[3usize] as u32),
1891 u8::wrapping_shr(a[4usize], b[4usize] as u32),
1892 u8::wrapping_shr(a[5usize], b[5usize] as u32),
1893 u8::wrapping_shr(a[6usize], b[6usize] as u32),
1894 u8::wrapping_shr(a[7usize], b[7usize] as u32),
1895 u8::wrapping_shr(a[8usize], b[8usize] as u32),
1896 u8::wrapping_shr(a[9usize], b[9usize] as u32),
1897 u8::wrapping_shr(a[10usize], b[10usize] as u32),
1898 u8::wrapping_shr(a[11usize], b[11usize] as u32),
1899 u8::wrapping_shr(a[12usize], b[12usize] as u32),
1900 u8::wrapping_shr(a[13usize], b[13usize] as u32),
1901 u8::wrapping_shr(a[14usize], b[14usize] as u32),
1902 u8::wrapping_shr(a[15usize], b[15usize] as u32),
1903 ]
1904 .simd_into(self)
1905 }
1906 #[inline(always)]
1907 fn reduce_max_u8x16(self, a: u8x16<Self>) -> u8 {
1908 crate::kernel!(
1909 #[inline(always)]
1910 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>) -> u8 {
1911 let reduced: __m128i = a.into();
1912 let shifted = _mm_srli_si128::<8>(reduced);
1913 let reduced = { _mm_max_epu8(reduced, shifted) };
1914 let shifted = _mm_srli_si128::<4>(reduced);
1915 let reduced = { _mm_max_epu8(reduced, shifted) };
1916 let shifted = _mm_srli_si128::<2>(reduced);
1917 let reduced = { _mm_max_epu8(reduced, shifted) };
1918 let shifted = _mm_srli_si128::<1>(reduced);
1919 let reduced = { _mm_max_epu8(reduced, shifted) };
1920 {
1921 let lanes: [u8; 16usize] = crate::transmute::checked_transmute_copy(&reduced);
1922 lanes[0]
1923 }
1924 }
1925 );
1926 kernel(self, a)
1927 }
1928 #[inline(always)]
1929 fn reduce_min_u8x16(self, a: u8x16<Self>) -> u8 {
1930 crate::kernel!(
1931 #[inline(always)]
1932 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>) -> u8 {
1933 let reduced: __m128i = a.into();
1934 let shifted = _mm_srli_si128::<8>(reduced);
1935 let reduced = { _mm_min_epu8(reduced, shifted) };
1936 let shifted = _mm_srli_si128::<4>(reduced);
1937 let reduced = { _mm_min_epu8(reduced, shifted) };
1938 let shifted = _mm_srli_si128::<2>(reduced);
1939 let reduced = { _mm_min_epu8(reduced, shifted) };
1940 let shifted = _mm_srli_si128::<1>(reduced);
1941 let reduced = { _mm_min_epu8(reduced, shifted) };
1942 {
1943 let lanes: [u8; 16usize] = crate::transmute::checked_transmute_copy(&reduced);
1944 lanes[0]
1945 }
1946 }
1947 );
1948 kernel(self, a)
1949 }
1950 #[inline(always)]
1951 fn reduce_sum_u8x16(self, a: u8x16<Self>) -> u8 {
1952 crate::kernel!(
1953 #[inline(always)]
1954 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>) -> u8 {
1955 let sum: __m128i = a.into();
1956 let sum = _mm_add_epi8(sum, _mm_srli_si128::<8>(sum));
1957 let sum = _mm_add_epi8(sum, _mm_srli_si128::<4>(sum));
1958 let sum = _mm_add_epi8(sum, _mm_srli_si128::<2>(sum));
1959 let sum = _mm_add_epi8(sum, _mm_srli_si128::<1>(sum));
1960 let lanes: [u8; 16usize] = crate::transmute::checked_transmute_copy(&sum);
1961 lanes[0]
1962 }
1963 );
1964 kernel(self, a)
1965 }
1966 #[inline(always)]
1967 fn reduce_product_u8x16(self, a: u8x16<Self>) -> u8 {
1968 crate::kernel!(
1969 #[inline(always)]
1970 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>) -> u8 {
1971 let value: __m128i = a.into();
1972 let product = _mm_mullo_epi16(
1973 _mm_unpacklo_epi8(value, value),
1974 _mm_unpackhi_epi8(value, value),
1975 );
1976 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b11_10_11_10>(product));
1977 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b01_01_01_01>(product));
1978 let product = _mm_mullo_epi16(product, _mm_srli_epi32::<16>(product));
1979 let lanes: [u8; 16usize] = crate::transmute::checked_transmute_copy(&product);
1980 lanes[0]
1981 }
1982 );
1983 kernel(self, a)
1984 }
1985 #[inline(always)]
1986 fn max_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1987 crate::kernel!(
1988 #[inline(always)]
1989 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
1990 _mm_max_epu8(a.into(), b.into()).simd_into(token)
1991 }
1992 );
1993 kernel(self, a, b)
1994 }
1995 #[inline(always)]
1996 fn min_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1997 crate::kernel!(
1998 #[inline(always)]
1999 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
2000 _mm_min_epu8(a.into(), b.into()).simd_into(token)
2001 }
2002 );
2003 kernel(self, a, b)
2004 }
2005 #[inline(always)]
2006 fn clamp_u8x16(self, a: u8x16<Self>, b: u8x16<Self>, c: u8x16<Self>) -> u8x16<Self> {
2007 debug_assert!(
2008 b.simd_le(c).all_true(),
2009 "clamp requires min <= max for all elements"
2010 );
2011 a.max(b).min(c)
2012 }
2013 #[inline(always)]
2014 fn simd_eq_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
2015 crate::kernel!(
2016 #[inline(always)]
2017 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> mask8x16<Sse4_2> {
2018 _mm_cmpeq_epi8(a.into(), b.into()).simd_into(token)
2019 }
2020 );
2021 kernel(self, a, b)
2022 }
2023 #[inline(always)]
2024 fn simd_lt_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
2025 crate::kernel!(
2026 #[inline(always)]
2027 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> mask8x16<Sse4_2> {
2028 {
2029 let sign_bit = _mm_set1_epi8(0x80u8.cast_signed());
2030 let lhs_signed = _mm_xor_si128(b.into(), sign_bit);
2031 let rhs_signed = _mm_xor_si128(a.into(), sign_bit);
2032 _mm_cmpgt_epi8(lhs_signed, rhs_signed)
2033 }
2034 .simd_into(token)
2035 }
2036 );
2037 kernel(self, a, b)
2038 }
2039 #[inline(always)]
2040 fn simd_le_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
2041 crate::kernel!(
2042 #[inline(always)]
2043 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> mask8x16<Sse4_2> {
2044 _mm_cmpeq_epi8(_mm_min_epu8(a.into(), b.into()), a.into()).simd_into(token)
2045 }
2046 );
2047 kernel(self, a, b)
2048 }
2049 #[inline(always)]
2050 fn zip_low_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2051 crate::kernel!(
2052 #[inline(always)]
2053 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
2054 _mm_unpacklo_epi8(a.into(), b.into()).simd_into(token)
2055 }
2056 );
2057 kernel(self, a, b)
2058 }
2059 #[inline(always)]
2060 fn zip_high_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2061 crate::kernel!(
2062 #[inline(always)]
2063 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
2064 _mm_unpackhi_epi8(a.into(), b.into()).simd_into(token)
2065 }
2066 );
2067 kernel(self, a, b)
2068 }
2069 #[inline(always)]
2070 fn unzip_low_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2071 crate::kernel!(
2072 #[inline(always)]
2073 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
2074 let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
2075 let t1 = _mm_shuffle_epi8(a.into(), mask);
2076 let t2 = _mm_shuffle_epi8(b.into(), mask);
2077 _mm_unpacklo_epi64(t1, t2).simd_into(token)
2078 }
2079 );
2080 kernel(self, a, b)
2081 }
2082 #[inline(always)]
2083 fn unzip_high_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2084 crate::kernel!(
2085 #[inline(always)]
2086 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>, b: u8x16<Sse4_2>) -> u8x16<Sse4_2> {
2087 let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
2088 let t1 = _mm_shuffle_epi8(a.into(), mask);
2089 let t2 = _mm_shuffle_epi8(b.into(), mask);
2090 _mm_unpackhi_epi64(t1, t2).simd_into(token)
2091 }
2092 );
2093 kernel(self, a, b)
2094 }
2095 #[inline(always)]
2096 fn interleave_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u8x16<Self>, u8x16<Self>) {
2097 (self.zip_low_u8x16(a, b), self.zip_high_u8x16(a, b))
2098 }
2099 #[inline(always)]
2100 fn deinterleave_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u8x16<Self>, u8x16<Self>) {
2101 crate::kernel!(
2102 #[inline(always)]
2103 fn kernel(
2104 token: Sse4_2,
2105 a: u8x16<Sse4_2>,
2106 b: u8x16<Sse4_2>,
2107 ) -> (u8x16<Sse4_2>, u8x16<Sse4_2>) {
2108 let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
2109 let a = _mm_shuffle_epi8(a.into(), mask);
2110 let b = _mm_shuffle_epi8(b.into(), mask);
2111 (
2112 _mm_unpacklo_epi64(a, b).simd_into(token),
2113 _mm_unpackhi_epi64(a, b).simd_into(token),
2114 )
2115 }
2116 );
2117 kernel(self, a, b)
2118 }
2119 #[inline(always)]
2120 fn select_u8x16(self, a: mask8x16<Self>, b: u8x16<Self>, c: u8x16<Self>) -> u8x16<Self> {
2121 crate::kernel!(
2122 #[inline(always)]
2123 fn kernel(
2124 token: Sse4_2,
2125 a: mask8x16<Sse4_2>,
2126 b: u8x16<Sse4_2>,
2127 c: u8x16<Sse4_2>,
2128 ) -> u8x16<Sse4_2> {
2129 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
2130 }
2131 );
2132 kernel(self, a, b, c)
2133 }
2134 #[inline(always)]
2135 fn combine_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x32<Self> {
2136 u8x32 {
2137 val: crate::support::Aligned256([a.val.0, b.val.0]),
2138 simd: self,
2139 }
2140 }
2141 #[inline(always)]
2142 fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16<Self>; 4usize] {
2143 crate::kernel!(
2144 #[inline(always)]
2145 fn kernel(token: Sse4_2, src: &[u8; 64usize]) -> [u8x16<Sse4_2>; 4usize] {
2146 let (chunks, []) = src.as_chunks::<16usize>() else {
2147 unreachable!()
2148 };
2149 let v0: __m128i =
2150 crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[0]);
2151 let v1: __m128i =
2152 crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[1]);
2153 let v2: __m128i =
2154 crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[2]);
2155 let v3: __m128i =
2156 crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[3]);
2157 let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
2158 let v0 = _mm_shuffle_epi8(v0, mask);
2159 let v1 = _mm_shuffle_epi8(v1, mask);
2160 let v2 = _mm_shuffle_epi8(v2, mask);
2161 let v3 = _mm_shuffle_epi8(v3, mask);
2162 let tmp0 = _mm_unpacklo_epi32(v0, v1);
2163 let tmp1 = _mm_unpackhi_epi32(v0, v1);
2164 let tmp2 = _mm_unpacklo_epi32(v2, v3);
2165 let tmp3 = _mm_unpackhi_epi32(v2, v3);
2166 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
2167 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
2168 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
2169 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
2170 [
2171 out0.simd_into(token),
2172 out1.simd_into(token),
2173 out2.simd_into(token),
2174 out3.simd_into(token),
2175 ]
2176 }
2177 );
2178 kernel(self, src)
2179 }
2180 #[inline(always)]
2181 fn store_four_interleaved_u8x16(
2182 self,
2183 vectors: [u8x16<Self>; 4usize],
2184 dest: &mut [u8; 64usize],
2185 ) -> () {
2186 crate::kernel!(
2187 #[inline(always)]
2188 fn kernel(
2189 token: Sse4_2,
2190 vectors: [u8x16<Sse4_2>; 4usize],
2191 dest: &mut [u8; 64usize],
2192 ) -> () {
2193 let _ = token;
2194 let v0: __m128i = vectors[0].into();
2195 let v1: __m128i = vectors[1].into();
2196 let v2: __m128i = vectors[2].into();
2197 let v3: __m128i = vectors[3].into();
2198 let tmp0 = _mm_unpacklo_epi32(v0, v1);
2199 let tmp1 = _mm_unpackhi_epi32(v0, v1);
2200 let tmp2 = _mm_unpacklo_epi32(v2, v3);
2201 let tmp3 = _mm_unpackhi_epi32(v2, v3);
2202 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
2203 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
2204 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
2205 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
2206 let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
2207 let out0 = _mm_shuffle_epi8(out0, mask);
2208 let out1 = _mm_shuffle_epi8(out1, mask);
2209 let out2 = _mm_shuffle_epi8(out2, mask);
2210 let out3 = _mm_shuffle_epi8(out3, mask);
2211 let (chunks, []) = dest.as_chunks_mut::<16usize>() else {
2212 unreachable!()
2213 };
2214 crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>(
2215 out0,
2216 &mut chunks[0],
2217 );
2218 crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>(
2219 out1,
2220 &mut chunks[1],
2221 );
2222 crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>(
2223 out2,
2224 &mut chunks[2],
2225 );
2226 crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>(
2227 out3,
2228 &mut chunks[3],
2229 );
2230 }
2231 );
2232 kernel(self, vectors, dest);
2233 }
2234 #[inline(always)]
2235 fn widen_u8x16(self, a: u8x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
2236 crate::kernel!(
2237 #[inline(always)]
2238 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>) -> (u16x8<Sse4_2>, u16x8<Sse4_2>) {
2239 let raw = a.into();
2240 (
2241 _mm_cvtepu8_epi16(raw).simd_into(token),
2242 _mm_cvtepu8_epi16(_mm_srli_si128::<8>(raw)).simd_into(token),
2243 )
2244 }
2245 );
2246 kernel(self, a)
2247 }
2248 #[inline(always)]
2249 fn widening_mul_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
2250 let (a0, a1) = self.widen_u8x16(a);
2251 let (b0, b1) = self.widen_u8x16(b);
2252 (self.mul_u16x8(a0, b0), self.mul_u16x8(a1, b1))
2253 }
2254 #[inline(always)]
2255 fn widening_add_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
2256 let (a0, a1) = self.widen_u8x16(a);
2257 let (b0, b1) = self.widen_u8x16(b);
2258 (self.add_u16x8(a0, b0), self.add_u16x8(a1, b1))
2259 }
2260 #[inline(always)]
2261 fn pairwise_widening_add_u8x16(self, a: u8x16<Self>) -> u16x8<Self> {
2262 crate::kernel!(
2263 #[inline(always)]
2264 fn kernel(token: Sse4_2, a: u8x16<Sse4_2>) -> u16x8<Sse4_2> {
2265 ({ _mm_maddubs_epi16(a.into(), _mm_set1_epi8(1)) }).simd_into(token)
2266 }
2267 );
2268 kernel(self, a)
2269 }
2270 #[inline(always)]
2271 fn pairwise_dot_product_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u16x8<Self> {
2272 let (low, high) = self.widening_mul_u8x16(a, b);
2273 let (even, odd) = self.deinterleave_u16x8(low, high);
2274 self.add_u16x8(even, odd)
2275 }
2276 #[inline(always)]
2277 fn splat_mask8x16(self, val: bool) -> mask8x16<Self> {
2278 crate::kernel!(
2279 #[inline(always)]
2280 fn kernel(token: Sse4_2, val: bool) -> mask8x16<Sse4_2> {
2281 let val: i8 = if val { !0 } else { 0 };
2282 _mm_set1_epi8(val).simd_into(token)
2283 }
2284 );
2285 kernel(self, val)
2286 }
2287 #[inline(always)]
2288 fn from_bitmask_mask8x16(self, bits: u64) -> mask8x16<Self> {
2289 crate::kernel!(
2290 #[inline(always)]
2291 fn kernel(token: Sse4_2, bits: u64) -> mask8x16<Sse4_2> {
2292 {
2293 let bit_bytes = _mm_cvtsi32_si128(bits as i32);
2294 let bit_bytes = _mm_shuffle_epi8(
2295 bit_bytes,
2296 _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1),
2297 );
2298 let bit_mask =
2299 _mm_setr_epi8(1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128);
2300 _mm_cmpeq_epi8(_mm_and_si128(bit_bytes, bit_mask), bit_mask)
2301 }
2302 .simd_into(token)
2303 }
2304 );
2305 kernel(self, bits)
2306 }
2307 #[inline(always)]
2308 fn to_bitmask_mask8x16(self, a: mask8x16<Self>) -> u64 {
2309 crate::kernel!(
2310 #[inline(always)]
2311 fn kernel(token: Sse4_2, a: mask8x16<Sse4_2>) -> u64 {
2312 _mm_movemask_epi8(a.into()) as u32 as u64
2313 }
2314 );
2315 kernel(self, a)
2316 }
2317 #[inline(always)]
2318 fn from_vector_mask8x16(self, vector: i8x16<Self>) -> mask8x16<Self> {
2319 mask8x16 {
2320 val: vector.val,
2321 simd: self,
2322 }
2323 }
2324 #[inline(always)]
2325 fn to_vector_mask8x16(self, a: mask8x16<Self>) -> i8x16<Self> {
2326 i8x16 {
2327 val: a.val,
2328 simd: self,
2329 }
2330 }
2331 #[inline(always)]
2332 fn set_mask8x16(self, a: &mut mask8x16<Self>, index: usize, value: bool) -> () {
2333 assert!(
2334 index < 16usize,
2335 "mask lane index {index} is out of bounds for {} lanes",
2336 16usize
2337 );
2338 let mut lanes: [i8; 16usize] = (*a).into();
2339 lanes[index] = if value { !0 } else { 0 };
2340 *a = lanes.simd_into(self);
2341 }
2342 #[inline(always)]
2343 fn rotate_elements_left_mask8x16<const OFFSET: usize>(
2344 self,
2345 a: mask8x16<Self>,
2346 ) -> mask8x16<Self> {
2347 let int = i8x16 {
2348 val: crate::transmute::checked_transmute_copy(&a.val),
2349 simd: self,
2350 };
2351 let rotated = int.rotate_elements_left::<OFFSET>();
2352 mask8x16 {
2353 val: crate::transmute::checked_transmute_copy(&rotated.val),
2354 simd: self,
2355 }
2356 }
2357 #[inline(always)]
2358 fn rotate_elements_right_mask8x16<const OFFSET: usize>(
2359 self,
2360 a: mask8x16<Self>,
2361 ) -> mask8x16<Self> {
2362 let int = i8x16 {
2363 val: crate::transmute::checked_transmute_copy(&a.val),
2364 simd: self,
2365 };
2366 let rotated = int.rotate_elements_right::<OFFSET>();
2367 mask8x16 {
2368 val: crate::transmute::checked_transmute_copy(&rotated.val),
2369 simd: self,
2370 }
2371 }
2372 #[inline(always)]
2373 fn and_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
2374 crate::kernel!(
2375 #[inline(always)]
2376 fn kernel(token: Sse4_2, a: mask8x16<Sse4_2>, b: mask8x16<Sse4_2>) -> mask8x16<Sse4_2> {
2377 _mm_and_si128(a.into(), b.into()).simd_into(token)
2378 }
2379 );
2380 kernel(self, a, b)
2381 }
2382 #[inline(always)]
2383 fn or_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
2384 crate::kernel!(
2385 #[inline(always)]
2386 fn kernel(token: Sse4_2, a: mask8x16<Sse4_2>, b: mask8x16<Sse4_2>) -> mask8x16<Sse4_2> {
2387 _mm_or_si128(a.into(), b.into()).simd_into(token)
2388 }
2389 );
2390 kernel(self, a, b)
2391 }
2392 #[inline(always)]
2393 fn xor_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
2394 crate::kernel!(
2395 #[inline(always)]
2396 fn kernel(token: Sse4_2, a: mask8x16<Sse4_2>, b: mask8x16<Sse4_2>) -> mask8x16<Sse4_2> {
2397 _mm_xor_si128(a.into(), b.into()).simd_into(token)
2398 }
2399 );
2400 kernel(self, a, b)
2401 }
2402 #[inline(always)]
2403 fn not_mask8x16(self, a: mask8x16<Self>) -> mask8x16<Self> {
2404 self.xor_mask8x16(a, self.splat_mask8x16(true))
2405 }
2406 #[inline(always)]
2407 fn reverse_mask8x16(self, a: mask8x16<Self>) -> mask8x16<Self> {
2408 let lanes = i8x16 {
2409 val: crate::transmute::checked_transmute_copy(&a.val),
2410 simd: self,
2411 };
2412 let reversed = self.reverse_i8x16(lanes);
2413 mask8x16 {
2414 val: crate::transmute::checked_transmute_copy(&reversed.val),
2415 simd: self,
2416 }
2417 }
2418 #[inline(always)]
2419 fn select_mask8x16(
2420 self,
2421 a: mask8x16<Self>,
2422 b: mask8x16<Self>,
2423 c: mask8x16<Self>,
2424 ) -> mask8x16<Self> {
2425 crate::kernel!(
2426 #[inline(always)]
2427 fn kernel(
2428 token: Sse4_2,
2429 a: mask8x16<Sse4_2>,
2430 b: mask8x16<Sse4_2>,
2431 c: mask8x16<Sse4_2>,
2432 ) -> mask8x16<Sse4_2> {
2433 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
2434 }
2435 );
2436 kernel(self, a, b, c)
2437 }
2438 #[inline(always)]
2439 fn simd_eq_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
2440 crate::kernel!(
2441 #[inline(always)]
2442 fn kernel(token: Sse4_2, a: mask8x16<Sse4_2>, b: mask8x16<Sse4_2>) -> mask8x16<Sse4_2> {
2443 _mm_cmpeq_epi8(a.into(), b.into()).simd_into(token)
2444 }
2445 );
2446 kernel(self, a, b)
2447 }
2448 #[inline(always)]
2449 fn any_true_mask8x16(self, a: mask8x16<Self>) -> bool {
2450 crate::kernel!(
2451 #[inline(always)]
2452 fn kernel(token: Sse4_2, a: mask8x16<Sse4_2>) -> bool {
2453 _mm_movemask_epi8(a.into()) as u32 != 0
2454 }
2455 );
2456 kernel(self, a)
2457 }
2458 #[inline(always)]
2459 fn all_true_mask8x16(self, a: mask8x16<Self>) -> bool {
2460 crate::kernel!(
2461 #[inline(always)]
2462 fn kernel(token: Sse4_2, a: mask8x16<Sse4_2>) -> bool {
2463 _mm_movemask_epi8(a.into()) as u32 == 0xffff
2464 }
2465 );
2466 kernel(self, a)
2467 }
2468 #[inline(always)]
2469 fn any_false_mask8x16(self, a: mask8x16<Self>) -> bool {
2470 crate::kernel!(
2471 #[inline(always)]
2472 fn kernel(token: Sse4_2, a: mask8x16<Sse4_2>) -> bool {
2473 _mm_movemask_epi8(a.into()) as u32 != 0xffff
2474 }
2475 );
2476 kernel(self, a)
2477 }
2478 #[inline(always)]
2479 fn all_false_mask8x16(self, a: mask8x16<Self>) -> bool {
2480 crate::kernel!(
2481 #[inline(always)]
2482 fn kernel(token: Sse4_2, a: mask8x16<Sse4_2>) -> bool {
2483 _mm_movemask_epi8(a.into()) as u32 == 0
2484 }
2485 );
2486 kernel(self, a)
2487 }
2488 #[inline(always)]
2489 fn combine_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x32<Self> {
2490 mask8x32 {
2491 val: crate::support::Aligned256([a.val.0, b.val.0]),
2492 simd: self,
2493 }
2494 }
2495 #[inline(always)]
2496 fn widen_mask8x16(self, a: mask8x16<Self>) -> (mask16x8<Self>, mask16x8<Self>) {
2497 crate::kernel!(
2498 #[inline(always)]
2499 fn kernel(token: Sse4_2, a: mask8x16<Sse4_2>) -> (mask16x8<Sse4_2>, mask16x8<Sse4_2>) {
2500 let raw = a.into();
2501 (
2502 _mm_unpacklo_epi8(raw, raw).simd_into(token),
2503 _mm_unpackhi_epi8(raw, raw).simd_into(token),
2504 )
2505 }
2506 );
2507 kernel(self, a)
2508 }
2509 #[inline(always)]
2510 fn abs_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2511 crate::kernel!(
2512 #[inline(always)]
2513 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
2514 _mm_abs_epi16(a.into()).simd_into(token)
2515 }
2516 );
2517 kernel(self, a)
2518 }
2519 #[inline(always)]
2520 fn splat_i16x8(self, val: i16) -> i16x8<Self> {
2521 crate::kernel!(
2522 #[inline(always)]
2523 fn kernel(token: Sse4_2, val: i16) -> i16x8<Sse4_2> {
2524 _mm_set1_epi16(val).simd_into(token)
2525 }
2526 );
2527 kernel(self, val)
2528 }
2529 #[inline(always)]
2530 fn reverse_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2531 let indices: u8x16<Self> =
2532 [14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1].simd_into(self);
2533 self.swizzle_dyn_i16x8(a, indices)
2534 }
2535 #[inline(always)]
2536 fn slide_i16x8<const SHIFT: usize>(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2537 if SHIFT >= 8usize {
2538 return b;
2539 }
2540 let result = dyn_alignr_128(
2541 self,
2542 Bytes::to_bytes(b).val.0,
2543 Bytes::to_bytes(a).val.0,
2544 SHIFT * 2usize,
2545 );
2546 Bytes::from_bytes(u8x16 {
2547 val: crate::support::Aligned128(result),
2548 simd: self,
2549 })
2550 }
2551 #[inline(always)]
2552 fn compress_i16x8(self, values: i16x8<Self>, mask: mask16x8<Self>) -> i16x8<Self> {
2553 let bits = self.to_bitmask_mask16x8(mask) as usize & 255;
2554 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0);
2555 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
2556 }
2557 #[inline(always)]
2558 fn compress_merge_i16x8(
2559 self,
2560 values: i16x8<Self>,
2561 mask: mask16x8<Self>,
2562 merge: i16x8<Self>,
2563 ) -> i16x8<Self> {
2564 let bits = self.to_bitmask_mask16x8(mask) as usize & 255;
2565 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0);
2566 let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control);
2567 let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0));
2568 Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed))
2569 }
2570 #[inline(always)]
2571 fn expand_i16x8(self, values: i16x8<Self>, mask: mask16x8<Self>) -> i16x8<Self> {
2572 let bits = self.to_bitmask_mask16x8(mask) as usize & 255;
2573 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0);
2574 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
2575 }
2576 #[inline(always)]
2577 fn expand_merge_i16x8(
2578 self,
2579 values: i16x8<Self>,
2580 mask: mask16x8<Self>,
2581 merge: i16x8<Self>,
2582 ) -> i16x8<Self> {
2583 let bits = self.to_bitmask_mask16x8(mask) as usize & 255;
2584 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0);
2585 let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control));
2586 self.select_i16x8(mask, expanded, merge)
2587 }
2588 #[inline(always)]
2589 fn count_ones_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2590 crate::kernel!(
2591 #[inline(always)]
2592 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
2593 let value = a.into();
2594 let nibble_mask = _mm_set1_epi8(0x0f);
2595 let lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4);
2596 let low = _mm_and_si128(value, nibble_mask);
2597 let high = _mm_and_si128(_mm_srli_epi16::<4>(value), nibble_mask);
2598 let byte_counts = _mm_add_epi8(
2599 _mm_shuffle_epi8(lookup, low),
2600 _mm_shuffle_epi8(lookup, high),
2601 );
2602 _mm_maddubs_epi16(byte_counts, _mm_set1_epi8(1)).simd_into(token)
2603 }
2604 );
2605 kernel(self, a)
2606 }
2607 #[inline(always)]
2608 fn count_zeros_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2609 self.count_ones_i16x8(self.not_i16x8(a))
2610 }
2611 #[inline(always)]
2612 fn add_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2613 crate::kernel!(
2614 #[inline(always)]
2615 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
2616 _mm_add_epi16(a.into(), b.into()).simd_into(token)
2617 }
2618 );
2619 kernel(self, a, b)
2620 }
2621 #[inline(always)]
2622 fn saturating_add_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2623 crate::kernel!(
2624 #[inline(always)]
2625 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
2626 _mm_adds_epi16(a.into(), b.into()).simd_into(token)
2627 }
2628 );
2629 kernel(self, a, b)
2630 }
2631 #[inline(always)]
2632 fn sub_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2633 crate::kernel!(
2634 #[inline(always)]
2635 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
2636 _mm_sub_epi16(a.into(), b.into()).simd_into(token)
2637 }
2638 );
2639 kernel(self, a, b)
2640 }
2641 #[inline(always)]
2642 fn saturating_sub_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2643 crate::kernel!(
2644 #[inline(always)]
2645 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
2646 _mm_subs_epi16(a.into(), b.into()).simd_into(token)
2647 }
2648 );
2649 kernel(self, a, b)
2650 }
2651 #[inline(always)]
2652 fn mul_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2653 crate::kernel!(
2654 #[inline(always)]
2655 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
2656 _mm_mullo_epi16(a.into(), b.into()).simd_into(token)
2657 }
2658 );
2659 kernel(self, a, b)
2660 }
2661 #[inline(always)]
2662 fn and_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2663 crate::kernel!(
2664 #[inline(always)]
2665 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
2666 _mm_and_si128(a.into(), b.into()).simd_into(token)
2667 }
2668 );
2669 kernel(self, a, b)
2670 }
2671 #[inline(always)]
2672 fn or_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2673 crate::kernel!(
2674 #[inline(always)]
2675 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
2676 _mm_or_si128(a.into(), b.into()).simd_into(token)
2677 }
2678 );
2679 kernel(self, a, b)
2680 }
2681 #[inline(always)]
2682 fn xor_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2683 crate::kernel!(
2684 #[inline(always)]
2685 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
2686 _mm_xor_si128(a.into(), b.into()).simd_into(token)
2687 }
2688 );
2689 kernel(self, a, b)
2690 }
2691 #[inline(always)]
2692 fn not_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2693 a ^ !0
2694 }
2695 #[inline(always)]
2696 fn shl_i16x8(self, a: i16x8<Self>, shift: u32) -> i16x8<Self> {
2697 crate::kernel!(
2698 #[inline(always)]
2699 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, shift: u32) -> i16x8<Sse4_2> {
2700 _mm_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
2701 }
2702 );
2703 kernel(self, a, shift)
2704 }
2705 #[inline(always)]
2706 fn shlv_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2707 [
2708 i16::wrapping_shl(a[0usize], b[0usize] as u32),
2709 i16::wrapping_shl(a[1usize], b[1usize] as u32),
2710 i16::wrapping_shl(a[2usize], b[2usize] as u32),
2711 i16::wrapping_shl(a[3usize], b[3usize] as u32),
2712 i16::wrapping_shl(a[4usize], b[4usize] as u32),
2713 i16::wrapping_shl(a[5usize], b[5usize] as u32),
2714 i16::wrapping_shl(a[6usize], b[6usize] as u32),
2715 i16::wrapping_shl(a[7usize], b[7usize] as u32),
2716 ]
2717 .simd_into(self)
2718 }
2719 #[inline(always)]
2720 fn shr_i16x8(self, a: i16x8<Self>, shift: u32) -> i16x8<Self> {
2721 crate::kernel!(
2722 #[inline(always)]
2723 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, shift: u32) -> i16x8<Sse4_2> {
2724 _mm_sra_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
2725 }
2726 );
2727 kernel(self, a, shift)
2728 }
2729 #[inline(always)]
2730 fn shrv_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2731 [
2732 i16::wrapping_shr(a[0usize], b[0usize] as u32),
2733 i16::wrapping_shr(a[1usize], b[1usize] as u32),
2734 i16::wrapping_shr(a[2usize], b[2usize] as u32),
2735 i16::wrapping_shr(a[3usize], b[3usize] as u32),
2736 i16::wrapping_shr(a[4usize], b[4usize] as u32),
2737 i16::wrapping_shr(a[5usize], b[5usize] as u32),
2738 i16::wrapping_shr(a[6usize], b[6usize] as u32),
2739 i16::wrapping_shr(a[7usize], b[7usize] as u32),
2740 ]
2741 .simd_into(self)
2742 }
2743 #[inline(always)]
2744 fn reduce_max_i16x8(self, a: i16x8<Self>) -> i16 {
2745 crate::kernel!(
2746 #[inline(always)]
2747 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>) -> i16 {
2748 let reduced: __m128i = a.into();
2749 let shifted = _mm_srli_si128::<8>(reduced);
2750 let reduced = { _mm_max_epi16(reduced, shifted) };
2751 let shifted = _mm_srli_si128::<4>(reduced);
2752 let reduced = { _mm_max_epi16(reduced, shifted) };
2753 let shifted = _mm_srli_si128::<2>(reduced);
2754 let reduced = { _mm_max_epi16(reduced, shifted) };
2755 {
2756 let lanes: [i16; 8usize] = crate::transmute::checked_transmute_copy(&reduced);
2757 lanes[0]
2758 }
2759 }
2760 );
2761 kernel(self, a)
2762 }
2763 #[inline(always)]
2764 fn reduce_min_i16x8(self, a: i16x8<Self>) -> i16 {
2765 crate::kernel!(
2766 #[inline(always)]
2767 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>) -> i16 {
2768 let reduced: __m128i = a.into();
2769 let shifted = _mm_srli_si128::<8>(reduced);
2770 let reduced = { _mm_min_epi16(reduced, shifted) };
2771 let shifted = _mm_srli_si128::<4>(reduced);
2772 let reduced = { _mm_min_epi16(reduced, shifted) };
2773 let shifted = _mm_srli_si128::<2>(reduced);
2774 let reduced = { _mm_min_epi16(reduced, shifted) };
2775 {
2776 let lanes: [i16; 8usize] = crate::transmute::checked_transmute_copy(&reduced);
2777 lanes[0]
2778 }
2779 }
2780 );
2781 kernel(self, a)
2782 }
2783 #[inline(always)]
2784 fn reduce_sum_i16x8(self, a: i16x8<Self>) -> i16 {
2785 crate::kernel!(
2786 #[inline(always)]
2787 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>) -> i16 {
2788 let sum: __m128i = a.into();
2789 let sum = _mm_add_epi16(sum, _mm_srli_si128::<8>(sum));
2790 let sum = _mm_add_epi16(sum, _mm_srli_si128::<4>(sum));
2791 let sum = _mm_add_epi16(sum, _mm_srli_si128::<2>(sum));
2792 let lanes: [i16; 8usize] = crate::transmute::checked_transmute_copy(&sum);
2793 lanes[0]
2794 }
2795 );
2796 kernel(self, a)
2797 }
2798 #[inline(always)]
2799 fn reduce_product_i16x8(self, a: i16x8<Self>) -> i16 {
2800 crate::kernel!(
2801 #[inline(always)]
2802 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>) -> i16 {
2803 let product: __m128i = a.into();
2804 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b11_10_11_10>(product));
2805 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b01_01_01_01>(product));
2806 let product = _mm_mullo_epi16(product, _mm_srli_epi32::<16>(product));
2807 let lanes: [i16; 8usize] = crate::transmute::checked_transmute_copy(&product);
2808 lanes[0]
2809 }
2810 );
2811 kernel(self, a)
2812 }
2813 #[inline(always)]
2814 fn max_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2815 crate::kernel!(
2816 #[inline(always)]
2817 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
2818 _mm_max_epi16(a.into(), b.into()).simd_into(token)
2819 }
2820 );
2821 kernel(self, a, b)
2822 }
2823 #[inline(always)]
2824 fn min_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2825 crate::kernel!(
2826 #[inline(always)]
2827 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
2828 _mm_min_epi16(a.into(), b.into()).simd_into(token)
2829 }
2830 );
2831 kernel(self, a, b)
2832 }
2833 #[inline(always)]
2834 fn clamp_i16x8(self, a: i16x8<Self>, b: i16x8<Self>, c: i16x8<Self>) -> i16x8<Self> {
2835 debug_assert!(
2836 b.simd_le(c).all_true(),
2837 "clamp requires min <= max for all elements"
2838 );
2839 a.max(b).min(c)
2840 }
2841 #[inline(always)]
2842 fn simd_eq_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
2843 crate::kernel!(
2844 #[inline(always)]
2845 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> mask16x8<Sse4_2> {
2846 _mm_cmpeq_epi16(a.into(), b.into()).simd_into(token)
2847 }
2848 );
2849 kernel(self, a, b)
2850 }
2851 #[inline(always)]
2852 fn simd_lt_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
2853 crate::kernel!(
2854 #[inline(always)]
2855 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> mask16x8<Sse4_2> {
2856 _mm_cmpgt_epi16(b.into(), a.into()).simd_into(token)
2857 }
2858 );
2859 kernel(self, a, b)
2860 }
2861 #[inline(always)]
2862 fn simd_le_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
2863 crate::kernel!(
2864 #[inline(always)]
2865 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> mask16x8<Sse4_2> {
2866 _mm_cmpeq_epi16(_mm_min_epi16(a.into(), b.into()), a.into()).simd_into(token)
2867 }
2868 );
2869 kernel(self, a, b)
2870 }
2871 #[inline(always)]
2872 fn zip_low_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2873 crate::kernel!(
2874 #[inline(always)]
2875 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
2876 _mm_unpacklo_epi16(a.into(), b.into()).simd_into(token)
2877 }
2878 );
2879 kernel(self, a, b)
2880 }
2881 #[inline(always)]
2882 fn zip_high_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2883 crate::kernel!(
2884 #[inline(always)]
2885 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
2886 _mm_unpackhi_epi16(a.into(), b.into()).simd_into(token)
2887 }
2888 );
2889 kernel(self, a, b)
2890 }
2891 #[inline(always)]
2892 fn unzip_low_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2893 crate::kernel!(
2894 #[inline(always)]
2895 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
2896 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
2897 let t1 = _mm_shuffle_epi8(a.into(), mask);
2898 let t2 = _mm_shuffle_epi8(b.into(), mask);
2899 _mm_unpacklo_epi64(t1, t2).simd_into(token)
2900 }
2901 );
2902 kernel(self, a, b)
2903 }
2904 #[inline(always)]
2905 fn unzip_high_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2906 crate::kernel!(
2907 #[inline(always)]
2908 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
2909 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
2910 let t1 = _mm_shuffle_epi8(a.into(), mask);
2911 let t2 = _mm_shuffle_epi8(b.into(), mask);
2912 _mm_unpackhi_epi64(t1, t2).simd_into(token)
2913 }
2914 );
2915 kernel(self, a, b)
2916 }
2917 #[inline(always)]
2918 fn interleave_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i16x8<Self>, i16x8<Self>) {
2919 (self.zip_low_i16x8(a, b), self.zip_high_i16x8(a, b))
2920 }
2921 #[inline(always)]
2922 fn deinterleave_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i16x8<Self>, i16x8<Self>) {
2923 crate::kernel!(
2924 #[inline(always)]
2925 fn kernel(
2926 token: Sse4_2,
2927 a: i16x8<Sse4_2>,
2928 b: i16x8<Sse4_2>,
2929 ) -> (i16x8<Sse4_2>, i16x8<Sse4_2>) {
2930 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
2931 let a = _mm_shuffle_epi8(a.into(), mask);
2932 let b = _mm_shuffle_epi8(b.into(), mask);
2933 (
2934 _mm_unpacklo_epi64(a, b).simd_into(token),
2935 _mm_unpackhi_epi64(a, b).simd_into(token),
2936 )
2937 }
2938 );
2939 kernel(self, a, b)
2940 }
2941 #[inline(always)]
2942 fn select_i16x8(self, a: mask16x8<Self>, b: i16x8<Self>, c: i16x8<Self>) -> i16x8<Self> {
2943 crate::kernel!(
2944 #[inline(always)]
2945 fn kernel(
2946 token: Sse4_2,
2947 a: mask16x8<Sse4_2>,
2948 b: i16x8<Sse4_2>,
2949 c: i16x8<Sse4_2>,
2950 ) -> i16x8<Sse4_2> {
2951 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
2952 }
2953 );
2954 kernel(self, a, b, c)
2955 }
2956 #[inline(always)]
2957 fn combine_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x16<Self> {
2958 i16x16 {
2959 val: crate::support::Aligned256([a.val.0, b.val.0]),
2960 simd: self,
2961 }
2962 }
2963 #[inline(always)]
2964 fn neg_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2965 crate::kernel!(
2966 #[inline(always)]
2967 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>) -> i16x8<Sse4_2> {
2968 _mm_sub_epi16(_mm_setzero_si128(), a.into()).simd_into(token)
2969 }
2970 );
2971 kernel(self, a)
2972 }
2973 #[inline(always)]
2974 fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8<Self>; 4usize] {
2975 crate::kernel!(
2976 #[inline(always)]
2977 fn kernel(token: Sse4_2, src: &[i16; 32usize]) -> [i16x8<Sse4_2>; 4usize] {
2978 let (chunks, []) = src.as_chunks::<8usize>() else {
2979 unreachable!()
2980 };
2981 let v0: __m128i =
2982 crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[0]);
2983 let v1: __m128i =
2984 crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[1]);
2985 let v2: __m128i =
2986 crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[2]);
2987 let v3: __m128i =
2988 crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[3]);
2989 let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15);
2990 let v0 = _mm_shuffle_epi8(v0, mask);
2991 let v1 = _mm_shuffle_epi8(v1, mask);
2992 let v2 = _mm_shuffle_epi8(v2, mask);
2993 let v3 = _mm_shuffle_epi8(v3, mask);
2994 let tmp0 = _mm_unpacklo_epi32(v0, v1);
2995 let tmp1 = _mm_unpackhi_epi32(v0, v1);
2996 let tmp2 = _mm_unpacklo_epi32(v2, v3);
2997 let tmp3 = _mm_unpackhi_epi32(v2, v3);
2998 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
2999 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
3000 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
3001 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
3002 [
3003 out0.simd_into(token),
3004 out1.simd_into(token),
3005 out2.simd_into(token),
3006 out3.simd_into(token),
3007 ]
3008 }
3009 );
3010 kernel(self, src)
3011 }
3012 #[inline(always)]
3013 fn store_four_interleaved_i16x8(
3014 self,
3015 vectors: [i16x8<Self>; 4usize],
3016 dest: &mut [i16; 32usize],
3017 ) -> () {
3018 crate::kernel!(
3019 #[inline(always)]
3020 fn kernel(
3021 token: Sse4_2,
3022 vectors: [i16x8<Sse4_2>; 4usize],
3023 dest: &mut [i16; 32usize],
3024 ) -> () {
3025 let _ = token;
3026 let v0: __m128i = vectors[0].into();
3027 let v1: __m128i = vectors[1].into();
3028 let v2: __m128i = vectors[2].into();
3029 let v3: __m128i = vectors[3].into();
3030 let tmp0 = _mm_unpacklo_epi32(v0, v1);
3031 let tmp1 = _mm_unpackhi_epi32(v0, v1);
3032 let tmp2 = _mm_unpacklo_epi32(v2, v3);
3033 let tmp3 = _mm_unpackhi_epi32(v2, v3);
3034 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
3035 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
3036 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
3037 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
3038 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
3039 let out0 = _mm_shuffle_epi8(out0, mask);
3040 let out1 = _mm_shuffle_epi8(out1, mask);
3041 let out2 = _mm_shuffle_epi8(out2, mask);
3042 let out3 = _mm_shuffle_epi8(out3, mask);
3043 let (chunks, []) = dest.as_chunks_mut::<8usize>() else {
3044 unreachable!()
3045 };
3046 crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>(
3047 out0,
3048 &mut chunks[0],
3049 );
3050 crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>(
3051 out1,
3052 &mut chunks[1],
3053 );
3054 crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>(
3055 out2,
3056 &mut chunks[2],
3057 );
3058 crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>(
3059 out3,
3060 &mut chunks[3],
3061 );
3062 }
3063 );
3064 kernel(self, vectors, dest);
3065 }
3066 #[inline(always)]
3067 fn widen_i16x8(self, a: i16x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
3068 crate::kernel!(
3069 #[inline(always)]
3070 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>) -> (i32x4<Sse4_2>, i32x4<Sse4_2>) {
3071 let raw = a.into();
3072 (
3073 _mm_cvtepi16_epi32(raw).simd_into(token),
3074 _mm_cvtepi16_epi32(_mm_srli_si128::<8>(raw)).simd_into(token),
3075 )
3076 }
3077 );
3078 kernel(self, a)
3079 }
3080 #[inline(always)]
3081 fn widening_mul_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
3082 let (a0, a1) = self.widen_i16x8(a);
3083 let (b0, b1) = self.widen_i16x8(b);
3084 (self.mul_i32x4(a0, b0), self.mul_i32x4(a1, b1))
3085 }
3086 #[inline(always)]
3087 fn widening_add_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
3088 let (a0, a1) = self.widen_i16x8(a);
3089 let (b0, b1) = self.widen_i16x8(b);
3090 (self.add_i32x4(a0, b0), self.add_i32x4(a1, b1))
3091 }
3092 #[inline(always)]
3093 fn pairwise_widening_add_i16x8(self, a: i16x8<Self>) -> i32x4<Self> {
3094 crate::kernel!(
3095 #[inline(always)]
3096 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>) -> i32x4<Sse4_2> {
3097 ({ _mm_madd_epi16(a.into(), _mm_set1_epi16(1)) }).simd_into(token)
3098 }
3099 );
3100 kernel(self, a)
3101 }
3102 #[inline(always)]
3103 fn pairwise_dot_product_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i32x4<Self> {
3104 crate::kernel!(
3105 #[inline(always)]
3106 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i32x4<Sse4_2> {
3107 _mm_madd_epi16(a.into(), b.into()).simd_into(token)
3108 }
3109 );
3110 kernel(self, a, b)
3111 }
3112 #[inline(always)]
3113 fn narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
3114 crate::kernel!(
3115 #[inline(always)]
3116 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i8x16<Sse4_2> {
3117 let mask = _mm_set1_epi16(0xff);
3118 _mm_packus_epi16(_mm_and_si128(a.into(), mask), _mm_and_si128(b.into(), mask))
3119 .simd_into(token)
3120 }
3121 );
3122 kernel(self, a, b)
3123 }
3124 #[inline(always)]
3125 fn saturating_narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
3126 crate::kernel!(
3127 #[inline(always)]
3128 fn kernel(token: Sse4_2, a: i16x8<Sse4_2>, b: i16x8<Sse4_2>) -> i8x16<Sse4_2> {
3129 _mm_packs_epi16(a.into(), b.into()).simd_into(token)
3130 }
3131 );
3132 kernel(self, a, b)
3133 }
3134 #[inline(always)]
3135 fn relaxed_narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
3136 debug_assert!(
3137 a.as_slice()
3138 .iter()
3139 .chain(b.as_slice())
3140 .all(|&value| { value >= i8::MIN as i16 && value <= i8::MAX as i16 }),
3141 "relaxed_narrow inputs must fit in the destination type",
3142 );
3143 self.saturating_narrow_i16x8(a, b)
3144 }
3145 #[inline(always)]
3146 fn splat_u16x8(self, val: u16) -> u16x8<Self> {
3147 crate::kernel!(
3148 #[inline(always)]
3149 fn kernel(token: Sse4_2, val: u16) -> u16x8<Sse4_2> {
3150 _mm_set1_epi16(val.cast_signed()).simd_into(token)
3151 }
3152 );
3153 kernel(self, val)
3154 }
3155 #[inline(always)]
3156 fn reverse_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
3157 let indices: u8x16<Self> =
3158 [14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1].simd_into(self);
3159 self.swizzle_dyn_u16x8(a, indices)
3160 }
3161 #[inline(always)]
3162 fn slide_u16x8<const SHIFT: usize>(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3163 if SHIFT >= 8usize {
3164 return b;
3165 }
3166 let result = dyn_alignr_128(
3167 self,
3168 Bytes::to_bytes(b).val.0,
3169 Bytes::to_bytes(a).val.0,
3170 SHIFT * 2usize,
3171 );
3172 Bytes::from_bytes(u8x16 {
3173 val: crate::support::Aligned128(result),
3174 simd: self,
3175 })
3176 }
3177 #[inline(always)]
3178 fn compress_u16x8(self, values: u16x8<Self>, mask: mask16x8<Self>) -> u16x8<Self> {
3179 let bits = self.to_bitmask_mask16x8(mask) as usize & 255;
3180 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0);
3181 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
3182 }
3183 #[inline(always)]
3184 fn compress_merge_u16x8(
3185 self,
3186 values: u16x8<Self>,
3187 mask: mask16x8<Self>,
3188 merge: u16x8<Self>,
3189 ) -> u16x8<Self> {
3190 let bits = self.to_bitmask_mask16x8(mask) as usize & 255;
3191 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_16[bits].0);
3192 let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control);
3193 let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0));
3194 Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed))
3195 }
3196 #[inline(always)]
3197 fn expand_u16x8(self, values: u16x8<Self>, mask: mask16x8<Self>) -> u16x8<Self> {
3198 let bits = self.to_bitmask_mask16x8(mask) as usize & 255;
3199 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0);
3200 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
3201 }
3202 #[inline(always)]
3203 fn expand_merge_u16x8(
3204 self,
3205 values: u16x8<Self>,
3206 mask: mask16x8<Self>,
3207 merge: u16x8<Self>,
3208 ) -> u16x8<Self> {
3209 let bits = self.to_bitmask_mask16x8(mask) as usize & 255;
3210 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_16[bits].0);
3211 let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control));
3212 self.select_u16x8(mask, expanded, merge)
3213 }
3214 #[inline(always)]
3215 fn count_ones_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
3216 crate::kernel!(
3217 #[inline(always)]
3218 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
3219 let value = a.into();
3220 let nibble_mask = _mm_set1_epi8(0x0f);
3221 let lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4);
3222 let low = _mm_and_si128(value, nibble_mask);
3223 let high = _mm_and_si128(_mm_srli_epi16::<4>(value), nibble_mask);
3224 let byte_counts = _mm_add_epi8(
3225 _mm_shuffle_epi8(lookup, low),
3226 _mm_shuffle_epi8(lookup, high),
3227 );
3228 _mm_maddubs_epi16(byte_counts, _mm_set1_epi8(1)).simd_into(token)
3229 }
3230 );
3231 kernel(self, a)
3232 }
3233 #[inline(always)]
3234 fn count_zeros_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
3235 self.count_ones_u16x8(self.not_u16x8(a))
3236 }
3237 #[inline(always)]
3238 fn add_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3239 crate::kernel!(
3240 #[inline(always)]
3241 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
3242 _mm_add_epi16(a.into(), b.into()).simd_into(token)
3243 }
3244 );
3245 kernel(self, a, b)
3246 }
3247 #[inline(always)]
3248 fn saturating_add_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3249 crate::kernel!(
3250 #[inline(always)]
3251 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
3252 _mm_adds_epu16(a.into(), b.into()).simd_into(token)
3253 }
3254 );
3255 kernel(self, a, b)
3256 }
3257 #[inline(always)]
3258 fn sub_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3259 crate::kernel!(
3260 #[inline(always)]
3261 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
3262 _mm_sub_epi16(a.into(), b.into()).simd_into(token)
3263 }
3264 );
3265 kernel(self, a, b)
3266 }
3267 #[inline(always)]
3268 fn saturating_sub_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3269 crate::kernel!(
3270 #[inline(always)]
3271 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
3272 _mm_subs_epu16(a.into(), b.into()).simd_into(token)
3273 }
3274 );
3275 kernel(self, a, b)
3276 }
3277 #[inline(always)]
3278 fn mul_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3279 crate::kernel!(
3280 #[inline(always)]
3281 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
3282 _mm_mullo_epi16(a.into(), b.into()).simd_into(token)
3283 }
3284 );
3285 kernel(self, a, b)
3286 }
3287 #[inline(always)]
3288 fn and_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3289 crate::kernel!(
3290 #[inline(always)]
3291 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
3292 _mm_and_si128(a.into(), b.into()).simd_into(token)
3293 }
3294 );
3295 kernel(self, a, b)
3296 }
3297 #[inline(always)]
3298 fn or_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3299 crate::kernel!(
3300 #[inline(always)]
3301 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
3302 _mm_or_si128(a.into(), b.into()).simd_into(token)
3303 }
3304 );
3305 kernel(self, a, b)
3306 }
3307 #[inline(always)]
3308 fn xor_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3309 crate::kernel!(
3310 #[inline(always)]
3311 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
3312 _mm_xor_si128(a.into(), b.into()).simd_into(token)
3313 }
3314 );
3315 kernel(self, a, b)
3316 }
3317 #[inline(always)]
3318 fn not_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
3319 a ^ !0
3320 }
3321 #[inline(always)]
3322 fn shl_u16x8(self, a: u16x8<Self>, shift: u32) -> u16x8<Self> {
3323 crate::kernel!(
3324 #[inline(always)]
3325 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, shift: u32) -> u16x8<Sse4_2> {
3326 _mm_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
3327 }
3328 );
3329 kernel(self, a, shift)
3330 }
3331 #[inline(always)]
3332 fn shlv_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3333 [
3334 u16::wrapping_shl(a[0usize], b[0usize] as u32),
3335 u16::wrapping_shl(a[1usize], b[1usize] as u32),
3336 u16::wrapping_shl(a[2usize], b[2usize] as u32),
3337 u16::wrapping_shl(a[3usize], b[3usize] as u32),
3338 u16::wrapping_shl(a[4usize], b[4usize] as u32),
3339 u16::wrapping_shl(a[5usize], b[5usize] as u32),
3340 u16::wrapping_shl(a[6usize], b[6usize] as u32),
3341 u16::wrapping_shl(a[7usize], b[7usize] as u32),
3342 ]
3343 .simd_into(self)
3344 }
3345 #[inline(always)]
3346 fn shr_u16x8(self, a: u16x8<Self>, shift: u32) -> u16x8<Self> {
3347 crate::kernel!(
3348 #[inline(always)]
3349 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, shift: u32) -> u16x8<Sse4_2> {
3350 _mm_srl_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
3351 }
3352 );
3353 kernel(self, a, shift)
3354 }
3355 #[inline(always)]
3356 fn shrv_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3357 [
3358 u16::wrapping_shr(a[0usize], b[0usize] as u32),
3359 u16::wrapping_shr(a[1usize], b[1usize] as u32),
3360 u16::wrapping_shr(a[2usize], b[2usize] as u32),
3361 u16::wrapping_shr(a[3usize], b[3usize] as u32),
3362 u16::wrapping_shr(a[4usize], b[4usize] as u32),
3363 u16::wrapping_shr(a[5usize], b[5usize] as u32),
3364 u16::wrapping_shr(a[6usize], b[6usize] as u32),
3365 u16::wrapping_shr(a[7usize], b[7usize] as u32),
3366 ]
3367 .simd_into(self)
3368 }
3369 #[inline(always)]
3370 fn reduce_max_u16x8(self, a: u16x8<Self>) -> u16 {
3371 crate::kernel!(
3372 #[inline(always)]
3373 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>) -> u16 {
3374 let reduced: __m128i = a.into();
3375 let shifted = _mm_srli_si128::<8>(reduced);
3376 let reduced = { _mm_max_epu16(reduced, shifted) };
3377 let shifted = _mm_srli_si128::<4>(reduced);
3378 let reduced = { _mm_max_epu16(reduced, shifted) };
3379 let shifted = _mm_srli_si128::<2>(reduced);
3380 let reduced = { _mm_max_epu16(reduced, shifted) };
3381 {
3382 let lanes: [u16; 8usize] = crate::transmute::checked_transmute_copy(&reduced);
3383 lanes[0]
3384 }
3385 }
3386 );
3387 kernel(self, a)
3388 }
3389 #[inline(always)]
3390 fn reduce_min_u16x8(self, a: u16x8<Self>) -> u16 {
3391 crate::kernel!(
3392 #[inline(always)]
3393 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>) -> u16 {
3394 let reduced: __m128i = a.into();
3395 let shifted = _mm_srli_si128::<8>(reduced);
3396 let reduced = { _mm_min_epu16(reduced, shifted) };
3397 let shifted = _mm_srli_si128::<4>(reduced);
3398 let reduced = { _mm_min_epu16(reduced, shifted) };
3399 let shifted = _mm_srli_si128::<2>(reduced);
3400 let reduced = { _mm_min_epu16(reduced, shifted) };
3401 {
3402 let lanes: [u16; 8usize] = crate::transmute::checked_transmute_copy(&reduced);
3403 lanes[0]
3404 }
3405 }
3406 );
3407 kernel(self, a)
3408 }
3409 #[inline(always)]
3410 fn reduce_sum_u16x8(self, a: u16x8<Self>) -> u16 {
3411 crate::kernel!(
3412 #[inline(always)]
3413 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>) -> u16 {
3414 let sum: __m128i = a.into();
3415 let sum = _mm_add_epi16(sum, _mm_srli_si128::<8>(sum));
3416 let sum = _mm_add_epi16(sum, _mm_srli_si128::<4>(sum));
3417 let sum = _mm_add_epi16(sum, _mm_srli_si128::<2>(sum));
3418 let lanes: [u16; 8usize] = crate::transmute::checked_transmute_copy(&sum);
3419 lanes[0]
3420 }
3421 );
3422 kernel(self, a)
3423 }
3424 #[inline(always)]
3425 fn reduce_product_u16x8(self, a: u16x8<Self>) -> u16 {
3426 crate::kernel!(
3427 #[inline(always)]
3428 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>) -> u16 {
3429 let product: __m128i = a.into();
3430 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b11_10_11_10>(product));
3431 let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b01_01_01_01>(product));
3432 let product = _mm_mullo_epi16(product, _mm_srli_epi32::<16>(product));
3433 let lanes: [u16; 8usize] = crate::transmute::checked_transmute_copy(&product);
3434 lanes[0]
3435 }
3436 );
3437 kernel(self, a)
3438 }
3439 #[inline(always)]
3440 fn max_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3441 crate::kernel!(
3442 #[inline(always)]
3443 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
3444 _mm_max_epu16(a.into(), b.into()).simd_into(token)
3445 }
3446 );
3447 kernel(self, a, b)
3448 }
3449 #[inline(always)]
3450 fn min_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3451 crate::kernel!(
3452 #[inline(always)]
3453 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
3454 _mm_min_epu16(a.into(), b.into()).simd_into(token)
3455 }
3456 );
3457 kernel(self, a, b)
3458 }
3459 #[inline(always)]
3460 fn clamp_u16x8(self, a: u16x8<Self>, b: u16x8<Self>, c: u16x8<Self>) -> u16x8<Self> {
3461 debug_assert!(
3462 b.simd_le(c).all_true(),
3463 "clamp requires min <= max for all elements"
3464 );
3465 a.max(b).min(c)
3466 }
3467 #[inline(always)]
3468 fn simd_eq_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
3469 crate::kernel!(
3470 #[inline(always)]
3471 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> mask16x8<Sse4_2> {
3472 _mm_cmpeq_epi16(a.into(), b.into()).simd_into(token)
3473 }
3474 );
3475 kernel(self, a, b)
3476 }
3477 #[inline(always)]
3478 fn simd_lt_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
3479 crate::kernel!(
3480 #[inline(always)]
3481 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> mask16x8<Sse4_2> {
3482 {
3483 let sign_bit = _mm_set1_epi16(0x8000u16.cast_signed());
3484 let lhs_signed = _mm_xor_si128(b.into(), sign_bit);
3485 let rhs_signed = _mm_xor_si128(a.into(), sign_bit);
3486 _mm_cmpgt_epi16(lhs_signed, rhs_signed)
3487 }
3488 .simd_into(token)
3489 }
3490 );
3491 kernel(self, a, b)
3492 }
3493 #[inline(always)]
3494 fn simd_le_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
3495 crate::kernel!(
3496 #[inline(always)]
3497 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> mask16x8<Sse4_2> {
3498 _mm_cmpeq_epi16(_mm_min_epu16(a.into(), b.into()), a.into()).simd_into(token)
3499 }
3500 );
3501 kernel(self, a, b)
3502 }
3503 #[inline(always)]
3504 fn zip_low_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3505 crate::kernel!(
3506 #[inline(always)]
3507 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
3508 _mm_unpacklo_epi16(a.into(), b.into()).simd_into(token)
3509 }
3510 );
3511 kernel(self, a, b)
3512 }
3513 #[inline(always)]
3514 fn zip_high_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3515 crate::kernel!(
3516 #[inline(always)]
3517 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
3518 _mm_unpackhi_epi16(a.into(), b.into()).simd_into(token)
3519 }
3520 );
3521 kernel(self, a, b)
3522 }
3523 #[inline(always)]
3524 fn unzip_low_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3525 crate::kernel!(
3526 #[inline(always)]
3527 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
3528 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
3529 let t1 = _mm_shuffle_epi8(a.into(), mask);
3530 let t2 = _mm_shuffle_epi8(b.into(), mask);
3531 _mm_unpacklo_epi64(t1, t2).simd_into(token)
3532 }
3533 );
3534 kernel(self, a, b)
3535 }
3536 #[inline(always)]
3537 fn unzip_high_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3538 crate::kernel!(
3539 #[inline(always)]
3540 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u16x8<Sse4_2> {
3541 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
3542 let t1 = _mm_shuffle_epi8(a.into(), mask);
3543 let t2 = _mm_shuffle_epi8(b.into(), mask);
3544 _mm_unpackhi_epi64(t1, t2).simd_into(token)
3545 }
3546 );
3547 kernel(self, a, b)
3548 }
3549 #[inline(always)]
3550 fn interleave_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u16x8<Self>, u16x8<Self>) {
3551 (self.zip_low_u16x8(a, b), self.zip_high_u16x8(a, b))
3552 }
3553 #[inline(always)]
3554 fn deinterleave_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u16x8<Self>, u16x8<Self>) {
3555 crate::kernel!(
3556 #[inline(always)]
3557 fn kernel(
3558 token: Sse4_2,
3559 a: u16x8<Sse4_2>,
3560 b: u16x8<Sse4_2>,
3561 ) -> (u16x8<Sse4_2>, u16x8<Sse4_2>) {
3562 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
3563 let a = _mm_shuffle_epi8(a.into(), mask);
3564 let b = _mm_shuffle_epi8(b.into(), mask);
3565 (
3566 _mm_unpacklo_epi64(a, b).simd_into(token),
3567 _mm_unpackhi_epi64(a, b).simd_into(token),
3568 )
3569 }
3570 );
3571 kernel(self, a, b)
3572 }
3573 #[inline(always)]
3574 fn select_u16x8(self, a: mask16x8<Self>, b: u16x8<Self>, c: u16x8<Self>) -> u16x8<Self> {
3575 crate::kernel!(
3576 #[inline(always)]
3577 fn kernel(
3578 token: Sse4_2,
3579 a: mask16x8<Sse4_2>,
3580 b: u16x8<Sse4_2>,
3581 c: u16x8<Sse4_2>,
3582 ) -> u16x8<Sse4_2> {
3583 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
3584 }
3585 );
3586 kernel(self, a, b, c)
3587 }
3588 #[inline(always)]
3589 fn combine_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x16<Self> {
3590 u16x16 {
3591 val: crate::support::Aligned256([a.val.0, b.val.0]),
3592 simd: self,
3593 }
3594 }
3595 #[inline(always)]
3596 fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8<Self>; 4usize] {
3597 crate::kernel!(
3598 #[inline(always)]
3599 fn kernel(token: Sse4_2, src: &[u16; 32usize]) -> [u16x8<Sse4_2>; 4usize] {
3600 let (chunks, []) = src.as_chunks::<8usize>() else {
3601 unreachable!()
3602 };
3603 let v0: __m128i =
3604 crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[0]);
3605 let v1: __m128i =
3606 crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[1]);
3607 let v2: __m128i =
3608 crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[2]);
3609 let v3: __m128i =
3610 crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[3]);
3611 let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15);
3612 let v0 = _mm_shuffle_epi8(v0, mask);
3613 let v1 = _mm_shuffle_epi8(v1, mask);
3614 let v2 = _mm_shuffle_epi8(v2, mask);
3615 let v3 = _mm_shuffle_epi8(v3, mask);
3616 let tmp0 = _mm_unpacklo_epi32(v0, v1);
3617 let tmp1 = _mm_unpackhi_epi32(v0, v1);
3618 let tmp2 = _mm_unpacklo_epi32(v2, v3);
3619 let tmp3 = _mm_unpackhi_epi32(v2, v3);
3620 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
3621 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
3622 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
3623 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
3624 [
3625 out0.simd_into(token),
3626 out1.simd_into(token),
3627 out2.simd_into(token),
3628 out3.simd_into(token),
3629 ]
3630 }
3631 );
3632 kernel(self, src)
3633 }
3634 #[inline(always)]
3635 fn store_four_interleaved_u16x8(
3636 self,
3637 vectors: [u16x8<Self>; 4usize],
3638 dest: &mut [u16; 32usize],
3639 ) -> () {
3640 crate::kernel!(
3641 #[inline(always)]
3642 fn kernel(
3643 token: Sse4_2,
3644 vectors: [u16x8<Sse4_2>; 4usize],
3645 dest: &mut [u16; 32usize],
3646 ) -> () {
3647 let _ = token;
3648 let v0: __m128i = vectors[0].into();
3649 let v1: __m128i = vectors[1].into();
3650 let v2: __m128i = vectors[2].into();
3651 let v3: __m128i = vectors[3].into();
3652 let tmp0 = _mm_unpacklo_epi32(v0, v1);
3653 let tmp1 = _mm_unpackhi_epi32(v0, v1);
3654 let tmp2 = _mm_unpacklo_epi32(v2, v3);
3655 let tmp3 = _mm_unpackhi_epi32(v2, v3);
3656 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
3657 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
3658 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
3659 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
3660 let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
3661 let out0 = _mm_shuffle_epi8(out0, mask);
3662 let out1 = _mm_shuffle_epi8(out1, mask);
3663 let out2 = _mm_shuffle_epi8(out2, mask);
3664 let out3 = _mm_shuffle_epi8(out3, mask);
3665 let (chunks, []) = dest.as_chunks_mut::<8usize>() else {
3666 unreachable!()
3667 };
3668 crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>(
3669 out0,
3670 &mut chunks[0],
3671 );
3672 crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>(
3673 out1,
3674 &mut chunks[1],
3675 );
3676 crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>(
3677 out2,
3678 &mut chunks[2],
3679 );
3680 crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>(
3681 out3,
3682 &mut chunks[3],
3683 );
3684 }
3685 );
3686 kernel(self, vectors, dest);
3687 }
3688 #[inline(always)]
3689 fn widen_u16x8(self, a: u16x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
3690 crate::kernel!(
3691 #[inline(always)]
3692 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>) -> (u32x4<Sse4_2>, u32x4<Sse4_2>) {
3693 let raw = a.into();
3694 (
3695 _mm_cvtepu16_epi32(raw).simd_into(token),
3696 _mm_cvtepu16_epi32(_mm_srli_si128::<8>(raw)).simd_into(token),
3697 )
3698 }
3699 );
3700 kernel(self, a)
3701 }
3702 #[inline(always)]
3703 fn widening_mul_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
3704 crate::kernel!(
3705 #[inline(always)]
3706 fn kernel(
3707 token: Sse4_2,
3708 a: u16x8<Sse4_2>,
3709 b: u16x8<Sse4_2>,
3710 ) -> (u32x4<Sse4_2>, u32x4<Sse4_2>) {
3711 let a = a.into();
3712 let b = b.into();
3713 let low = _mm_mullo_epi16(a, b);
3714 let high = _mm_mulhi_epu16(a, b);
3715 let result_low = _mm_unpacklo_epi16(low, high);
3716 let result_high = _mm_unpackhi_epi16(low, high);
3717 (result_low.simd_into(token), result_high.simd_into(token))
3718 }
3719 );
3720 kernel(self, a, b)
3721 }
3722 #[inline(always)]
3723 fn widening_add_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
3724 let (a0, a1) = self.widen_u16x8(a);
3725 let (b0, b1) = self.widen_u16x8(b);
3726 (self.add_u32x4(a0, b0), self.add_u32x4(a1, b1))
3727 }
3728 #[inline(always)]
3729 fn pairwise_widening_add_u16x8(self, a: u16x8<Self>) -> u32x4<Self> {
3730 crate::kernel!(
3731 #[inline(always)]
3732 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>) -> u32x4<Sse4_2> {
3733 ({
3734 let a = a.into();
3735 _mm_add_epi32(
3736 _mm_and_si128(a, _mm_set1_epi32(65535)),
3737 _mm_srli_epi32::<16>(a),
3738 )
3739 })
3740 .simd_into(token)
3741 }
3742 );
3743 kernel(self, a)
3744 }
3745 #[inline(always)]
3746 fn pairwise_dot_product_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u32x4<Self> {
3747 let (low, high) = self.widening_mul_u16x8(a, b);
3748 let (even, odd) = self.deinterleave_u32x4(low, high);
3749 self.add_u32x4(even, odd)
3750 }
3751 #[inline(always)]
3752 fn narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
3753 crate::kernel!(
3754 #[inline(always)]
3755 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u8x16<Sse4_2> {
3756 let mask = _mm_set1_epi16(0xff);
3757 _mm_packus_epi16(_mm_and_si128(a.into(), mask), _mm_and_si128(b.into(), mask))
3758 .simd_into(token)
3759 }
3760 );
3761 kernel(self, a, b)
3762 }
3763 #[inline(always)]
3764 fn saturating_narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
3765 crate::kernel!(
3766 #[inline(always)]
3767 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u8x16<Sse4_2> {
3768 {
3769 let max = _mm_set1_epi16(i16::MAX);
3770 _mm_packus_epi16(_mm_min_epu16(a.into(), max), _mm_min_epu16(b.into(), max))
3771 }
3772 .simd_into(token)
3773 }
3774 );
3775 kernel(self, a, b)
3776 }
3777 #[inline(always)]
3778 fn relaxed_narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
3779 crate::kernel!(
3780 #[inline(always)]
3781 fn kernel(token: Sse4_2, a: u16x8<Sse4_2>, b: u16x8<Sse4_2>) -> u8x16<Sse4_2> {
3782 debug_assert!(
3783 a.as_slice()
3784 .iter()
3785 .chain(b.as_slice())
3786 .all(|&value| { value >= u8::MIN as u16 && value <= u8::MAX as u16 }),
3787 "relaxed_narrow inputs must fit in the destination type",
3788 );
3789 _mm_packus_epi16(a.into(), b.into()).simd_into(token)
3790 }
3791 );
3792 kernel(self, a, b)
3793 }
3794 #[inline(always)]
3795 fn splat_mask16x8(self, val: bool) -> mask16x8<Self> {
3796 crate::kernel!(
3797 #[inline(always)]
3798 fn kernel(token: Sse4_2, val: bool) -> mask16x8<Sse4_2> {
3799 let val: i16 = if val { !0 } else { 0 };
3800 _mm_set1_epi16(val).simd_into(token)
3801 }
3802 );
3803 kernel(self, val)
3804 }
3805 #[inline(always)]
3806 fn from_bitmask_mask16x8(self, bits: u64) -> mask16x8<Self> {
3807 crate::kernel!(
3808 #[inline(always)]
3809 fn kernel(token: Sse4_2, bits: u64) -> mask16x8<Sse4_2> {
3810 {
3811 let bit_lanes = _mm_set1_epi16(bits as i16);
3812 let bit_mask = _mm_setr_epi16(1, 2, 4, 8, 16, 32, 64, 128);
3813 _mm_cmpeq_epi16(_mm_and_si128(bit_lanes, bit_mask), bit_mask)
3814 }
3815 .simd_into(token)
3816 }
3817 );
3818 kernel(self, bits)
3819 }
3820 #[inline(always)]
3821 fn to_bitmask_mask16x8(self, a: mask16x8<Self>) -> u64 {
3822 crate::kernel!(
3823 #[inline(always)]
3824 fn kernel(token: Sse4_2, a: mask16x8<Sse4_2>) -> u64 {
3825 {
3826 let packed = _mm_packs_epi16(a.into(), a.into());
3827 _mm_movemask_epi8(packed) as u8 as u64
3828 }
3829 }
3830 );
3831 kernel(self, a)
3832 }
3833 #[inline(always)]
3834 fn from_vector_mask16x8(self, vector: i16x8<Self>) -> mask16x8<Self> {
3835 mask16x8 {
3836 val: vector.val,
3837 simd: self,
3838 }
3839 }
3840 #[inline(always)]
3841 fn to_vector_mask16x8(self, a: mask16x8<Self>) -> i16x8<Self> {
3842 i16x8 {
3843 val: a.val,
3844 simd: self,
3845 }
3846 }
3847 #[inline(always)]
3848 fn set_mask16x8(self, a: &mut mask16x8<Self>, index: usize, value: bool) -> () {
3849 assert!(
3850 index < 8usize,
3851 "mask lane index {index} is out of bounds for {} lanes",
3852 8usize
3853 );
3854 let mut lanes: [i16; 8usize] = (*a).into();
3855 lanes[index] = if value { !0 } else { 0 };
3856 *a = lanes.simd_into(self);
3857 }
3858 #[inline(always)]
3859 fn rotate_elements_left_mask16x8<const OFFSET: usize>(
3860 self,
3861 a: mask16x8<Self>,
3862 ) -> mask16x8<Self> {
3863 let int = i16x8 {
3864 val: crate::transmute::checked_transmute_copy(&a.val),
3865 simd: self,
3866 };
3867 let rotated = int.rotate_elements_left::<OFFSET>();
3868 mask16x8 {
3869 val: crate::transmute::checked_transmute_copy(&rotated.val),
3870 simd: self,
3871 }
3872 }
3873 #[inline(always)]
3874 fn rotate_elements_right_mask16x8<const OFFSET: usize>(
3875 self,
3876 a: mask16x8<Self>,
3877 ) -> mask16x8<Self> {
3878 let int = i16x8 {
3879 val: crate::transmute::checked_transmute_copy(&a.val),
3880 simd: self,
3881 };
3882 let rotated = int.rotate_elements_right::<OFFSET>();
3883 mask16x8 {
3884 val: crate::transmute::checked_transmute_copy(&rotated.val),
3885 simd: self,
3886 }
3887 }
3888 #[inline(always)]
3889 fn and_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
3890 crate::kernel!(
3891 #[inline(always)]
3892 fn kernel(token: Sse4_2, a: mask16x8<Sse4_2>, b: mask16x8<Sse4_2>) -> mask16x8<Sse4_2> {
3893 _mm_and_si128(a.into(), b.into()).simd_into(token)
3894 }
3895 );
3896 kernel(self, a, b)
3897 }
3898 #[inline(always)]
3899 fn or_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
3900 crate::kernel!(
3901 #[inline(always)]
3902 fn kernel(token: Sse4_2, a: mask16x8<Sse4_2>, b: mask16x8<Sse4_2>) -> mask16x8<Sse4_2> {
3903 _mm_or_si128(a.into(), b.into()).simd_into(token)
3904 }
3905 );
3906 kernel(self, a, b)
3907 }
3908 #[inline(always)]
3909 fn xor_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
3910 crate::kernel!(
3911 #[inline(always)]
3912 fn kernel(token: Sse4_2, a: mask16x8<Sse4_2>, b: mask16x8<Sse4_2>) -> mask16x8<Sse4_2> {
3913 _mm_xor_si128(a.into(), b.into()).simd_into(token)
3914 }
3915 );
3916 kernel(self, a, b)
3917 }
3918 #[inline(always)]
3919 fn not_mask16x8(self, a: mask16x8<Self>) -> mask16x8<Self> {
3920 self.xor_mask16x8(a, self.splat_mask16x8(true))
3921 }
3922 #[inline(always)]
3923 fn reverse_mask16x8(self, a: mask16x8<Self>) -> mask16x8<Self> {
3924 let lanes = i16x8 {
3925 val: crate::transmute::checked_transmute_copy(&a.val),
3926 simd: self,
3927 };
3928 let reversed = self.reverse_i16x8(lanes);
3929 mask16x8 {
3930 val: crate::transmute::checked_transmute_copy(&reversed.val),
3931 simd: self,
3932 }
3933 }
3934 #[inline(always)]
3935 fn select_mask16x8(
3936 self,
3937 a: mask16x8<Self>,
3938 b: mask16x8<Self>,
3939 c: mask16x8<Self>,
3940 ) -> mask16x8<Self> {
3941 crate::kernel!(
3942 #[inline(always)]
3943 fn kernel(
3944 token: Sse4_2,
3945 a: mask16x8<Sse4_2>,
3946 b: mask16x8<Sse4_2>,
3947 c: mask16x8<Sse4_2>,
3948 ) -> mask16x8<Sse4_2> {
3949 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
3950 }
3951 );
3952 kernel(self, a, b, c)
3953 }
3954 #[inline(always)]
3955 fn simd_eq_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
3956 crate::kernel!(
3957 #[inline(always)]
3958 fn kernel(token: Sse4_2, a: mask16x8<Sse4_2>, b: mask16x8<Sse4_2>) -> mask16x8<Sse4_2> {
3959 _mm_cmpeq_epi16(a.into(), b.into()).simd_into(token)
3960 }
3961 );
3962 kernel(self, a, b)
3963 }
3964 #[inline(always)]
3965 fn any_true_mask16x8(self, a: mask16x8<Self>) -> bool {
3966 crate::kernel!(
3967 #[inline(always)]
3968 fn kernel(token: Sse4_2, a: mask16x8<Sse4_2>) -> bool {
3969 _mm_movemask_epi8(a.into()) as u32 != 0
3970 }
3971 );
3972 kernel(self, a)
3973 }
3974 #[inline(always)]
3975 fn all_true_mask16x8(self, a: mask16x8<Self>) -> bool {
3976 crate::kernel!(
3977 #[inline(always)]
3978 fn kernel(token: Sse4_2, a: mask16x8<Sse4_2>) -> bool {
3979 _mm_movemask_epi8(a.into()) as u32 == 0xffff
3980 }
3981 );
3982 kernel(self, a)
3983 }
3984 #[inline(always)]
3985 fn any_false_mask16x8(self, a: mask16x8<Self>) -> bool {
3986 crate::kernel!(
3987 #[inline(always)]
3988 fn kernel(token: Sse4_2, a: mask16x8<Sse4_2>) -> bool {
3989 _mm_movemask_epi8(a.into()) as u32 != 0xffff
3990 }
3991 );
3992 kernel(self, a)
3993 }
3994 #[inline(always)]
3995 fn all_false_mask16x8(self, a: mask16x8<Self>) -> bool {
3996 crate::kernel!(
3997 #[inline(always)]
3998 fn kernel(token: Sse4_2, a: mask16x8<Sse4_2>) -> bool {
3999 _mm_movemask_epi8(a.into()) as u32 == 0
4000 }
4001 );
4002 kernel(self, a)
4003 }
4004 #[inline(always)]
4005 fn combine_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x16<Self> {
4006 mask16x16 {
4007 val: crate::support::Aligned256([a.val.0, b.val.0]),
4008 simd: self,
4009 }
4010 }
4011 #[inline(always)]
4012 fn widen_mask16x8(self, a: mask16x8<Self>) -> (mask32x4<Self>, mask32x4<Self>) {
4013 crate::kernel!(
4014 #[inline(always)]
4015 fn kernel(token: Sse4_2, a: mask16x8<Sse4_2>) -> (mask32x4<Sse4_2>, mask32x4<Sse4_2>) {
4016 let raw = a.into();
4017 (
4018 _mm_unpacklo_epi16(raw, raw).simd_into(token),
4019 _mm_unpackhi_epi16(raw, raw).simd_into(token),
4020 )
4021 }
4022 );
4023 kernel(self, a)
4024 }
4025 #[inline(always)]
4026 fn narrow_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask8x16<Self> {
4027 crate::kernel!(
4028 #[inline(always)]
4029 fn kernel(token: Sse4_2, a: mask16x8<Sse4_2>, b: mask16x8<Sse4_2>) -> mask8x16<Sse4_2> {
4030 _mm_packs_epi16(a.into(), b.into()).simd_into(token)
4031 }
4032 );
4033 kernel(self, a, b)
4034 }
4035 #[inline(always)]
4036 fn abs_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4037 crate::kernel!(
4038 #[inline(always)]
4039 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
4040 _mm_abs_epi32(a.into()).simd_into(token)
4041 }
4042 );
4043 kernel(self, a)
4044 }
4045 #[inline(always)]
4046 fn splat_i32x4(self, val: i32) -> i32x4<Self> {
4047 crate::kernel!(
4048 #[inline(always)]
4049 fn kernel(token: Sse4_2, val: i32) -> i32x4<Sse4_2> {
4050 _mm_set1_epi32(val).simd_into(token)
4051 }
4052 );
4053 kernel(self, val)
4054 }
4055 #[inline(always)]
4056 fn reverse_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4057 let indices: u8x16<Self> =
4058 [12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3].simd_into(self);
4059 self.swizzle_dyn_i32x4(a, indices)
4060 }
4061 #[inline(always)]
4062 fn slide_i32x4<const SHIFT: usize>(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4063 if SHIFT >= 4usize {
4064 return b;
4065 }
4066 let result = dyn_alignr_128(
4067 self,
4068 Bytes::to_bytes(b).val.0,
4069 Bytes::to_bytes(a).val.0,
4070 SHIFT * 4usize,
4071 );
4072 Bytes::from_bytes(u8x16 {
4073 val: crate::support::Aligned128(result),
4074 simd: self,
4075 })
4076 }
4077 #[inline(always)]
4078 fn compress_i32x4(self, values: i32x4<Self>, mask: mask32x4<Self>) -> i32x4<Self> {
4079 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
4080 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0);
4081 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
4082 }
4083 #[inline(always)]
4084 fn compress_merge_i32x4(
4085 self,
4086 values: i32x4<Self>,
4087 mask: mask32x4<Self>,
4088 merge: i32x4<Self>,
4089 ) -> i32x4<Self> {
4090 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
4091 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0);
4092 let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control);
4093 let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0));
4094 Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed))
4095 }
4096 #[inline(always)]
4097 fn expand_i32x4(self, values: i32x4<Self>, mask: mask32x4<Self>) -> i32x4<Self> {
4098 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
4099 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0);
4100 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
4101 }
4102 #[inline(always)]
4103 fn expand_merge_i32x4(
4104 self,
4105 values: i32x4<Self>,
4106 mask: mask32x4<Self>,
4107 merge: i32x4<Self>,
4108 ) -> i32x4<Self> {
4109 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
4110 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0);
4111 let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control));
4112 self.select_i32x4(mask, expanded, merge)
4113 }
4114 #[inline(always)]
4115 fn count_ones_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4116 crate::kernel!(
4117 #[inline(always)]
4118 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
4119 let value = a.into();
4120 let nibble_mask = _mm_set1_epi8(0x0f);
4121 let lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4);
4122 let low = _mm_and_si128(value, nibble_mask);
4123 let high = _mm_and_si128(_mm_srli_epi16::<4>(value), nibble_mask);
4124 let byte_counts = _mm_add_epi8(
4125 _mm_shuffle_epi8(lookup, low),
4126 _mm_shuffle_epi8(lookup, high),
4127 );
4128 _mm_madd_epi16(
4129 _mm_maddubs_epi16(byte_counts, _mm_set1_epi8(1)),
4130 _mm_set1_epi16(1),
4131 )
4132 .simd_into(token)
4133 }
4134 );
4135 kernel(self, a)
4136 }
4137 #[inline(always)]
4138 fn count_zeros_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4139 self.count_ones_i32x4(self.not_i32x4(a))
4140 }
4141 #[inline(always)]
4142 fn add_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4143 crate::kernel!(
4144 #[inline(always)]
4145 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
4146 _mm_add_epi32(a.into(), b.into()).simd_into(token)
4147 }
4148 );
4149 kernel(self, a, b)
4150 }
4151 #[inline(always)]
4152 fn saturating_add_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4153 crate::kernel!(
4154 #[inline(always)]
4155 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
4156 let a = a.into();
4157 let b = b.into();
4158 let wrapped = _mm_add_epi32(a, b);
4159 let overflow = _mm_xor_si128(_mm_cmpgt_epi32(a, wrapped), b);
4160 let bound = _mm_xor_si128(_mm_srai_epi32::<31>(a), _mm_set1_epi32(i32::MAX));
4161 let result = _mm_blendv_ps(
4162 _mm_castsi128_ps(wrapped),
4163 _mm_castsi128_ps(bound),
4164 _mm_castsi128_ps(overflow),
4165 );
4166 _mm_castps_si128(result).simd_into(token)
4167 }
4168 );
4169 kernel(self, a, b)
4170 }
4171 #[inline(always)]
4172 fn sub_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4173 crate::kernel!(
4174 #[inline(always)]
4175 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
4176 _mm_sub_epi32(a.into(), b.into()).simd_into(token)
4177 }
4178 );
4179 kernel(self, a, b)
4180 }
4181 #[inline(always)]
4182 fn saturating_sub_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4183 crate::kernel!(
4184 #[inline(always)]
4185 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
4186 let a = a.into();
4187 let b = b.into();
4188 let wrapped = _mm_sub_epi32(a, b);
4189 let overflow = _mm_xor_si128(_mm_cmpgt_epi32(wrapped, a), b);
4190 let bound = _mm_xor_si128(_mm_srai_epi32::<31>(a), _mm_set1_epi32(i32::MAX));
4191 let result = _mm_blendv_ps(
4192 _mm_castsi128_ps(wrapped),
4193 _mm_castsi128_ps(bound),
4194 _mm_castsi128_ps(overflow),
4195 );
4196 _mm_castps_si128(result).simd_into(token)
4197 }
4198 );
4199 kernel(self, a, b)
4200 }
4201 #[inline(always)]
4202 fn mul_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4203 crate::kernel!(
4204 #[inline(always)]
4205 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
4206 _mm_mullo_epi32(a.into(), b.into()).simd_into(token)
4207 }
4208 );
4209 kernel(self, a, b)
4210 }
4211 #[inline(always)]
4212 fn and_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4213 crate::kernel!(
4214 #[inline(always)]
4215 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
4216 _mm_and_si128(a.into(), b.into()).simd_into(token)
4217 }
4218 );
4219 kernel(self, a, b)
4220 }
4221 #[inline(always)]
4222 fn or_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4223 crate::kernel!(
4224 #[inline(always)]
4225 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
4226 _mm_or_si128(a.into(), b.into()).simd_into(token)
4227 }
4228 );
4229 kernel(self, a, b)
4230 }
4231 #[inline(always)]
4232 fn xor_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4233 crate::kernel!(
4234 #[inline(always)]
4235 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
4236 _mm_xor_si128(a.into(), b.into()).simd_into(token)
4237 }
4238 );
4239 kernel(self, a, b)
4240 }
4241 #[inline(always)]
4242 fn not_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4243 a ^ !0
4244 }
4245 #[inline(always)]
4246 fn shl_i32x4(self, a: i32x4<Self>, shift: u32) -> i32x4<Self> {
4247 crate::kernel!(
4248 #[inline(always)]
4249 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, shift: u32) -> i32x4<Sse4_2> {
4250 _mm_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
4251 }
4252 );
4253 kernel(self, a, shift)
4254 }
4255 #[inline(always)]
4256 fn shlv_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4257 [
4258 i32::wrapping_shl(a[0usize], b[0usize] as u32),
4259 i32::wrapping_shl(a[1usize], b[1usize] as u32),
4260 i32::wrapping_shl(a[2usize], b[2usize] as u32),
4261 i32::wrapping_shl(a[3usize], b[3usize] as u32),
4262 ]
4263 .simd_into(self)
4264 }
4265 #[inline(always)]
4266 fn shr_i32x4(self, a: i32x4<Self>, shift: u32) -> i32x4<Self> {
4267 crate::kernel!(
4268 #[inline(always)]
4269 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, shift: u32) -> i32x4<Sse4_2> {
4270 _mm_sra_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
4271 }
4272 );
4273 kernel(self, a, shift)
4274 }
4275 #[inline(always)]
4276 fn shrv_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4277 [
4278 i32::wrapping_shr(a[0usize], b[0usize] as u32),
4279 i32::wrapping_shr(a[1usize], b[1usize] as u32),
4280 i32::wrapping_shr(a[2usize], b[2usize] as u32),
4281 i32::wrapping_shr(a[3usize], b[3usize] as u32),
4282 ]
4283 .simd_into(self)
4284 }
4285 #[inline(always)]
4286 fn reduce_max_i32x4(self, a: i32x4<Self>) -> i32 {
4287 crate::kernel!(
4288 #[inline(always)]
4289 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>) -> i32 {
4290 let reduced: __m128i = a.into();
4291 let shifted = _mm_srli_si128::<8>(reduced);
4292 let reduced = { _mm_max_epi32(reduced, shifted) };
4293 let shifted = _mm_srli_si128::<4>(reduced);
4294 let reduced = { _mm_max_epi32(reduced, shifted) };
4295 {
4296 let lanes: [i32; 4usize] = crate::transmute::checked_transmute_copy(&reduced);
4297 lanes[0]
4298 }
4299 }
4300 );
4301 kernel(self, a)
4302 }
4303 #[inline(always)]
4304 fn reduce_min_i32x4(self, a: i32x4<Self>) -> i32 {
4305 crate::kernel!(
4306 #[inline(always)]
4307 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>) -> i32 {
4308 let reduced: __m128i = a.into();
4309 let shifted = _mm_srli_si128::<8>(reduced);
4310 let reduced = { _mm_min_epi32(reduced, shifted) };
4311 let shifted = _mm_srli_si128::<4>(reduced);
4312 let reduced = { _mm_min_epi32(reduced, shifted) };
4313 {
4314 let lanes: [i32; 4usize] = crate::transmute::checked_transmute_copy(&reduced);
4315 lanes[0]
4316 }
4317 }
4318 );
4319 kernel(self, a)
4320 }
4321 #[inline(always)]
4322 fn reduce_sum_i32x4(self, a: i32x4<Self>) -> i32 {
4323 crate::kernel!(
4324 #[inline(always)]
4325 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>) -> i32 {
4326 let sum: __m128i = a.into();
4327 let sum = _mm_add_epi32(sum, _mm_srli_si128::<8>(sum));
4328 let sum = _mm_add_epi32(sum, _mm_srli_si128::<4>(sum));
4329 let lanes: [i32; 4usize] = crate::transmute::checked_transmute_copy(&sum);
4330 lanes[0]
4331 }
4332 );
4333 kernel(self, a)
4334 }
4335 #[inline(always)]
4336 fn reduce_product_i32x4(self, a: i32x4<Self>) -> i32 {
4337 let mul_level_0: [i32; 2usize] = [
4338 a[0usize].wrapping_mul(a[1usize]),
4339 a[2usize].wrapping_mul(a[3usize]),
4340 ];
4341 let mul_level_1: [i32; 1usize] = [mul_level_0[0usize].wrapping_mul(mul_level_0[1usize])];
4342 mul_level_1[0]
4343 }
4344 #[inline(always)]
4345 fn max_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4346 crate::kernel!(
4347 #[inline(always)]
4348 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
4349 _mm_max_epi32(a.into(), b.into()).simd_into(token)
4350 }
4351 );
4352 kernel(self, a, b)
4353 }
4354 #[inline(always)]
4355 fn min_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4356 crate::kernel!(
4357 #[inline(always)]
4358 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
4359 _mm_min_epi32(a.into(), b.into()).simd_into(token)
4360 }
4361 );
4362 kernel(self, a, b)
4363 }
4364 #[inline(always)]
4365 fn clamp_i32x4(self, a: i32x4<Self>, b: i32x4<Self>, c: i32x4<Self>) -> i32x4<Self> {
4366 debug_assert!(
4367 b.simd_le(c).all_true(),
4368 "clamp requires min <= max for all elements"
4369 );
4370 a.max(b).min(c)
4371 }
4372 #[inline(always)]
4373 fn simd_eq_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
4374 crate::kernel!(
4375 #[inline(always)]
4376 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> mask32x4<Sse4_2> {
4377 _mm_cmpeq_epi32(a.into(), b.into()).simd_into(token)
4378 }
4379 );
4380 kernel(self, a, b)
4381 }
4382 #[inline(always)]
4383 fn simd_lt_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
4384 crate::kernel!(
4385 #[inline(always)]
4386 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> mask32x4<Sse4_2> {
4387 _mm_cmpgt_epi32(b.into(), a.into()).simd_into(token)
4388 }
4389 );
4390 kernel(self, a, b)
4391 }
4392 #[inline(always)]
4393 fn simd_le_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
4394 crate::kernel!(
4395 #[inline(always)]
4396 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> mask32x4<Sse4_2> {
4397 _mm_cmpeq_epi32(_mm_min_epi32(a.into(), b.into()), a.into()).simd_into(token)
4398 }
4399 );
4400 kernel(self, a, b)
4401 }
4402 #[inline(always)]
4403 fn zip_low_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4404 crate::kernel!(
4405 #[inline(always)]
4406 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
4407 _mm_unpacklo_epi32(a.into(), b.into()).simd_into(token)
4408 }
4409 );
4410 kernel(self, a, b)
4411 }
4412 #[inline(always)]
4413 fn zip_high_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4414 crate::kernel!(
4415 #[inline(always)]
4416 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
4417 _mm_unpackhi_epi32(a.into(), b.into()).simd_into(token)
4418 }
4419 );
4420 kernel(self, a, b)
4421 }
4422 #[inline(always)]
4423 fn unzip_low_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4424 crate::kernel!(
4425 #[inline(always)]
4426 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
4427 let t1 = _mm_shuffle_epi32::<0b11_01_10_00>(a.into());
4428 let t2 = _mm_shuffle_epi32::<0b11_01_10_00>(b.into());
4429 _mm_unpacklo_epi64(t1, t2).simd_into(token)
4430 }
4431 );
4432 kernel(self, a, b)
4433 }
4434 #[inline(always)]
4435 fn unzip_high_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4436 crate::kernel!(
4437 #[inline(always)]
4438 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
4439 let t1 = _mm_shuffle_epi32::<0b11_01_10_00>(a.into());
4440 let t2 = _mm_shuffle_epi32::<0b11_01_10_00>(b.into());
4441 _mm_unpackhi_epi64(t1, t2).simd_into(token)
4442 }
4443 );
4444 kernel(self, a, b)
4445 }
4446 #[inline(always)]
4447 fn interleave_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i32x4<Self>, i32x4<Self>) {
4448 (self.zip_low_i32x4(a, b), self.zip_high_i32x4(a, b))
4449 }
4450 #[inline(always)]
4451 fn deinterleave_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i32x4<Self>, i32x4<Self>) {
4452 (self.unzip_low_i32x4(a, b), self.unzip_high_i32x4(a, b))
4453 }
4454 #[inline(always)]
4455 fn select_i32x4(self, a: mask32x4<Self>, b: i32x4<Self>, c: i32x4<Self>) -> i32x4<Self> {
4456 crate::kernel!(
4457 #[inline(always)]
4458 fn kernel(
4459 token: Sse4_2,
4460 a: mask32x4<Sse4_2>,
4461 b: i32x4<Sse4_2>,
4462 c: i32x4<Sse4_2>,
4463 ) -> i32x4<Sse4_2> {
4464 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
4465 }
4466 );
4467 kernel(self, a, b, c)
4468 }
4469 #[inline(always)]
4470 fn combine_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x8<Self> {
4471 i32x8 {
4472 val: crate::support::Aligned256([a.val.0, b.val.0]),
4473 simd: self,
4474 }
4475 }
4476 #[inline(always)]
4477 fn neg_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4478 crate::kernel!(
4479 #[inline(always)]
4480 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>) -> i32x4<Sse4_2> {
4481 _mm_sub_epi32(_mm_setzero_si128(), a.into()).simd_into(token)
4482 }
4483 );
4484 kernel(self, a)
4485 }
4486 #[inline(always)]
4487 fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4<Self>; 4usize] {
4488 crate::kernel!(
4489 #[inline(always)]
4490 fn kernel(token: Sse4_2, src: &[i32; 16usize]) -> [i32x4<Sse4_2>; 4usize] {
4491 let (chunks, []) = src.as_chunks::<4usize>() else {
4492 unreachable!()
4493 };
4494 let v0: __m128i =
4495 crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[0]);
4496 let v1: __m128i =
4497 crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[1]);
4498 let v2: __m128i =
4499 crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[2]);
4500 let v3: __m128i =
4501 crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[3]);
4502 let tmp0 = _mm_unpacklo_epi32(v0, v1);
4503 let tmp1 = _mm_unpackhi_epi32(v0, v1);
4504 let tmp2 = _mm_unpacklo_epi32(v2, v3);
4505 let tmp3 = _mm_unpackhi_epi32(v2, v3);
4506 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
4507 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
4508 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
4509 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
4510 [
4511 out0.simd_into(token),
4512 out1.simd_into(token),
4513 out2.simd_into(token),
4514 out3.simd_into(token),
4515 ]
4516 }
4517 );
4518 kernel(self, src)
4519 }
4520 #[inline(always)]
4521 fn store_four_interleaved_i32x4(
4522 self,
4523 vectors: [i32x4<Self>; 4usize],
4524 dest: &mut [i32; 16usize],
4525 ) -> () {
4526 crate::kernel!(
4527 #[inline(always)]
4528 fn kernel(
4529 token: Sse4_2,
4530 vectors: [i32x4<Sse4_2>; 4usize],
4531 dest: &mut [i32; 16usize],
4532 ) -> () {
4533 let _ = token;
4534 let v0: __m128i = vectors[0].into();
4535 let v1: __m128i = vectors[1].into();
4536 let v2: __m128i = vectors[2].into();
4537 let v3: __m128i = vectors[3].into();
4538 let tmp0 = _mm_unpacklo_epi32(v0, v1);
4539 let tmp1 = _mm_unpackhi_epi32(v0, v1);
4540 let tmp2 = _mm_unpacklo_epi32(v2, v3);
4541 let tmp3 = _mm_unpackhi_epi32(v2, v3);
4542 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
4543 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
4544 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
4545 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
4546 let (chunks, []) = dest.as_chunks_mut::<4usize>() else {
4547 unreachable!()
4548 };
4549 crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>(
4550 out0,
4551 &mut chunks[0],
4552 );
4553 crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>(
4554 out1,
4555 &mut chunks[1],
4556 );
4557 crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>(
4558 out2,
4559 &mut chunks[2],
4560 );
4561 crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>(
4562 out3,
4563 &mut chunks[3],
4564 );
4565 }
4566 );
4567 kernel(self, vectors, dest);
4568 }
4569 #[inline(always)]
4570 fn widen_i32x4(self, a: i32x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
4571 crate::kernel!(
4572 #[inline(always)]
4573 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>) -> (i64x2<Sse4_2>, i64x2<Sse4_2>) {
4574 let raw = a.into();
4575 (
4576 _mm_cvtepi32_epi64(raw).simd_into(token),
4577 _mm_cvtepi32_epi64(_mm_srli_si128::<8>(raw)).simd_into(token),
4578 )
4579 }
4580 );
4581 kernel(self, a)
4582 }
4583 #[inline(always)]
4584 fn widening_mul_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
4585 crate::kernel!(
4586 #[inline(always)]
4587 fn kernel(
4588 token: Sse4_2,
4589 a: i32x4<Sse4_2>,
4590 b: i32x4<Sse4_2>,
4591 ) -> (i64x2<Sse4_2>, i64x2<Sse4_2>) {
4592 let a = a.into();
4593 let b = b.into();
4594 let low = _mm_mul_epi32(a, b);
4595 let high =
4596 _mm_mul_epi32(_mm_shuffle_epi32::<0xf5>(a), _mm_shuffle_epi32::<0xf5>(b));
4597 let result_low = _mm_unpacklo_epi64(low, high);
4598 let result_high = _mm_unpackhi_epi64(low, high);
4599 (result_low.simd_into(token), result_high.simd_into(token))
4600 }
4601 );
4602 kernel(self, a, b)
4603 }
4604 #[inline(always)]
4605 fn widening_add_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
4606 let (a0, a1) = self.widen_i32x4(a);
4607 let (b0, b1) = self.widen_i32x4(b);
4608 (self.add_i64x2(a0, b0), self.add_i64x2(a1, b1))
4609 }
4610 #[inline(always)]
4611 fn pairwise_widening_add_i32x4(self, a: i32x4<Self>) -> i64x2<Self> {
4612 crate::kernel!(
4613 #[inline(always)]
4614 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>) -> i64x2<Sse4_2> {
4615 ({
4616 let y = _mm_xor_si128(a.into(), _mm_set1_epi32(i32::MIN));
4617 _mm_add_epi64(
4618 _mm_or_si128(y, _mm_set1_epi64x(-0x1_0000_0000)),
4619 _mm_srli_epi64::<32>(y),
4620 )
4621 })
4622 .simd_into(token)
4623 }
4624 );
4625 kernel(self, a)
4626 }
4627 #[inline(always)]
4628 fn pairwise_dot_product_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i64x2<Self> {
4629 let (low, high) = self.widening_mul_i32x4(a, b);
4630 let (even, odd) = self.deinterleave_i64x2(low, high);
4631 self.add_i64x2(even, odd)
4632 }
4633 #[inline(always)]
4634 fn narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
4635 crate::kernel!(
4636 #[inline(always)]
4637 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i16x8<Sse4_2> {
4638 let mask = _mm_set1_epi32(0xffff);
4639 _mm_packus_epi32(_mm_and_si128(a.into(), mask), _mm_and_si128(b.into(), mask))
4640 .simd_into(token)
4641 }
4642 );
4643 kernel(self, a, b)
4644 }
4645 #[inline(always)]
4646 fn saturating_narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
4647 crate::kernel!(
4648 #[inline(always)]
4649 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>, b: i32x4<Sse4_2>) -> i16x8<Sse4_2> {
4650 _mm_packs_epi32(a.into(), b.into()).simd_into(token)
4651 }
4652 );
4653 kernel(self, a, b)
4654 }
4655 #[inline(always)]
4656 fn relaxed_narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
4657 debug_assert!(
4658 a.as_slice()
4659 .iter()
4660 .chain(b.as_slice())
4661 .all(|&value| { value >= i16::MIN as i32 && value <= i16::MAX as i32 }),
4662 "relaxed_narrow inputs must fit in the destination type",
4663 );
4664 self.saturating_narrow_i32x4(a, b)
4665 }
4666 #[inline(always)]
4667 fn cvt_f32_i32x4(self, a: i32x4<Self>) -> f32x4<Self> {
4668 crate::kernel!(
4669 #[inline(always)]
4670 fn kernel(token: Sse4_2, a: i32x4<Sse4_2>) -> f32x4<Sse4_2> {
4671 _mm_cvtepi32_ps(a.into()).simd_into(token)
4672 }
4673 );
4674 kernel(self, a)
4675 }
4676 #[inline(always)]
4677 fn splat_u32x4(self, val: u32) -> u32x4<Self> {
4678 crate::kernel!(
4679 #[inline(always)]
4680 fn kernel(token: Sse4_2, val: u32) -> u32x4<Sse4_2> {
4681 _mm_set1_epi32(val.cast_signed()).simd_into(token)
4682 }
4683 );
4684 kernel(self, val)
4685 }
4686 #[inline(always)]
4687 fn reverse_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
4688 let indices: u8x16<Self> =
4689 [12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3].simd_into(self);
4690 self.swizzle_dyn_u32x4(a, indices)
4691 }
4692 #[inline(always)]
4693 fn slide_u32x4<const SHIFT: usize>(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4694 if SHIFT >= 4usize {
4695 return b;
4696 }
4697 let result = dyn_alignr_128(
4698 self,
4699 Bytes::to_bytes(b).val.0,
4700 Bytes::to_bytes(a).val.0,
4701 SHIFT * 4usize,
4702 );
4703 Bytes::from_bytes(u8x16 {
4704 val: crate::support::Aligned128(result),
4705 simd: self,
4706 })
4707 }
4708 #[inline(always)]
4709 fn compress_u32x4(self, values: u32x4<Self>, mask: mask32x4<Self>) -> u32x4<Self> {
4710 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
4711 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0);
4712 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
4713 }
4714 #[inline(always)]
4715 fn compress_merge_u32x4(
4716 self,
4717 values: u32x4<Self>,
4718 mask: mask32x4<Self>,
4719 merge: u32x4<Self>,
4720 ) -> u32x4<Self> {
4721 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
4722 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_32[bits].0);
4723 let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control);
4724 let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0));
4725 Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed))
4726 }
4727 #[inline(always)]
4728 fn expand_u32x4(self, values: u32x4<Self>, mask: mask32x4<Self>) -> u32x4<Self> {
4729 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
4730 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0);
4731 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
4732 }
4733 #[inline(always)]
4734 fn expand_merge_u32x4(
4735 self,
4736 values: u32x4<Self>,
4737 mask: mask32x4<Self>,
4738 merge: u32x4<Self>,
4739 ) -> u32x4<Self> {
4740 let bits = self.to_bitmask_mask32x4(mask) as usize & 15;
4741 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_32[bits].0);
4742 let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control));
4743 self.select_u32x4(mask, expanded, merge)
4744 }
4745 #[inline(always)]
4746 fn count_ones_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
4747 crate::kernel!(
4748 #[inline(always)]
4749 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
4750 let value = a.into();
4751 let nibble_mask = _mm_set1_epi8(0x0f);
4752 let lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4);
4753 let low = _mm_and_si128(value, nibble_mask);
4754 let high = _mm_and_si128(_mm_srli_epi16::<4>(value), nibble_mask);
4755 let byte_counts = _mm_add_epi8(
4756 _mm_shuffle_epi8(lookup, low),
4757 _mm_shuffle_epi8(lookup, high),
4758 );
4759 _mm_madd_epi16(
4760 _mm_maddubs_epi16(byte_counts, _mm_set1_epi8(1)),
4761 _mm_set1_epi16(1),
4762 )
4763 .simd_into(token)
4764 }
4765 );
4766 kernel(self, a)
4767 }
4768 #[inline(always)]
4769 fn count_zeros_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
4770 self.count_ones_u32x4(self.not_u32x4(a))
4771 }
4772 #[inline(always)]
4773 fn add_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4774 crate::kernel!(
4775 #[inline(always)]
4776 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
4777 _mm_add_epi32(a.into(), b.into()).simd_into(token)
4778 }
4779 );
4780 kernel(self, a, b)
4781 }
4782 #[inline(always)]
4783 fn saturating_add_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4784 crate::kernel!(
4785 #[inline(always)]
4786 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
4787 let a = a.into();
4788 let b = b.into();
4789 let threshold = _mm_xor_si128(b, _mm_set1_epi32(-1));
4790 _mm_add_epi32(_mm_min_epu32(a, threshold), b).simd_into(token)
4791 }
4792 );
4793 kernel(self, a, b)
4794 }
4795 #[inline(always)]
4796 fn sub_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4797 crate::kernel!(
4798 #[inline(always)]
4799 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
4800 _mm_sub_epi32(a.into(), b.into()).simd_into(token)
4801 }
4802 );
4803 kernel(self, a, b)
4804 }
4805 #[inline(always)]
4806 fn saturating_sub_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4807 crate::kernel!(
4808 #[inline(always)]
4809 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
4810 let a = a.into();
4811 let b = b.into();
4812 _mm_sub_epi32(_mm_max_epu32(a, b), b).simd_into(token)
4813 }
4814 );
4815 kernel(self, a, b)
4816 }
4817 #[inline(always)]
4818 fn mul_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4819 crate::kernel!(
4820 #[inline(always)]
4821 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
4822 _mm_mullo_epi32(a.into(), b.into()).simd_into(token)
4823 }
4824 );
4825 kernel(self, a, b)
4826 }
4827 #[inline(always)]
4828 fn and_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4829 crate::kernel!(
4830 #[inline(always)]
4831 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
4832 _mm_and_si128(a.into(), b.into()).simd_into(token)
4833 }
4834 );
4835 kernel(self, a, b)
4836 }
4837 #[inline(always)]
4838 fn or_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4839 crate::kernel!(
4840 #[inline(always)]
4841 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
4842 _mm_or_si128(a.into(), b.into()).simd_into(token)
4843 }
4844 );
4845 kernel(self, a, b)
4846 }
4847 #[inline(always)]
4848 fn xor_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4849 crate::kernel!(
4850 #[inline(always)]
4851 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
4852 _mm_xor_si128(a.into(), b.into()).simd_into(token)
4853 }
4854 );
4855 kernel(self, a, b)
4856 }
4857 #[inline(always)]
4858 fn not_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
4859 a ^ !0
4860 }
4861 #[inline(always)]
4862 fn shl_u32x4(self, a: u32x4<Self>, shift: u32) -> u32x4<Self> {
4863 crate::kernel!(
4864 #[inline(always)]
4865 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, shift: u32) -> u32x4<Sse4_2> {
4866 _mm_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
4867 }
4868 );
4869 kernel(self, a, shift)
4870 }
4871 #[inline(always)]
4872 fn shlv_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4873 [
4874 u32::wrapping_shl(a[0usize], b[0usize] as u32),
4875 u32::wrapping_shl(a[1usize], b[1usize] as u32),
4876 u32::wrapping_shl(a[2usize], b[2usize] as u32),
4877 u32::wrapping_shl(a[3usize], b[3usize] as u32),
4878 ]
4879 .simd_into(self)
4880 }
4881 #[inline(always)]
4882 fn shr_u32x4(self, a: u32x4<Self>, shift: u32) -> u32x4<Self> {
4883 crate::kernel!(
4884 #[inline(always)]
4885 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, shift: u32) -> u32x4<Sse4_2> {
4886 _mm_srl_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
4887 }
4888 );
4889 kernel(self, a, shift)
4890 }
4891 #[inline(always)]
4892 fn shrv_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4893 [
4894 u32::wrapping_shr(a[0usize], b[0usize] as u32),
4895 u32::wrapping_shr(a[1usize], b[1usize] as u32),
4896 u32::wrapping_shr(a[2usize], b[2usize] as u32),
4897 u32::wrapping_shr(a[3usize], b[3usize] as u32),
4898 ]
4899 .simd_into(self)
4900 }
4901 #[inline(always)]
4902 fn reduce_max_u32x4(self, a: u32x4<Self>) -> u32 {
4903 crate::kernel!(
4904 #[inline(always)]
4905 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>) -> u32 {
4906 let reduced: __m128i = a.into();
4907 let shifted = _mm_srli_si128::<8>(reduced);
4908 let reduced = { _mm_max_epu32(reduced, shifted) };
4909 let shifted = _mm_srli_si128::<4>(reduced);
4910 let reduced = { _mm_max_epu32(reduced, shifted) };
4911 {
4912 let lanes: [u32; 4usize] = crate::transmute::checked_transmute_copy(&reduced);
4913 lanes[0]
4914 }
4915 }
4916 );
4917 kernel(self, a)
4918 }
4919 #[inline(always)]
4920 fn reduce_min_u32x4(self, a: u32x4<Self>) -> u32 {
4921 crate::kernel!(
4922 #[inline(always)]
4923 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>) -> u32 {
4924 let reduced: __m128i = a.into();
4925 let shifted = _mm_srli_si128::<8>(reduced);
4926 let reduced = { _mm_min_epu32(reduced, shifted) };
4927 let shifted = _mm_srli_si128::<4>(reduced);
4928 let reduced = { _mm_min_epu32(reduced, shifted) };
4929 {
4930 let lanes: [u32; 4usize] = crate::transmute::checked_transmute_copy(&reduced);
4931 lanes[0]
4932 }
4933 }
4934 );
4935 kernel(self, a)
4936 }
4937 #[inline(always)]
4938 fn reduce_sum_u32x4(self, a: u32x4<Self>) -> u32 {
4939 crate::kernel!(
4940 #[inline(always)]
4941 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>) -> u32 {
4942 let sum: __m128i = a.into();
4943 let sum = _mm_add_epi32(sum, _mm_srli_si128::<8>(sum));
4944 let sum = _mm_add_epi32(sum, _mm_srli_si128::<4>(sum));
4945 let lanes: [u32; 4usize] = crate::transmute::checked_transmute_copy(&sum);
4946 lanes[0]
4947 }
4948 );
4949 kernel(self, a)
4950 }
4951 #[inline(always)]
4952 fn reduce_product_u32x4(self, a: u32x4<Self>) -> u32 {
4953 let mul_level_0: [u32; 2usize] = [
4954 a[0usize].wrapping_mul(a[1usize]),
4955 a[2usize].wrapping_mul(a[3usize]),
4956 ];
4957 let mul_level_1: [u32; 1usize] = [mul_level_0[0usize].wrapping_mul(mul_level_0[1usize])];
4958 mul_level_1[0]
4959 }
4960 #[inline(always)]
4961 fn max_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4962 crate::kernel!(
4963 #[inline(always)]
4964 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
4965 _mm_max_epu32(a.into(), b.into()).simd_into(token)
4966 }
4967 );
4968 kernel(self, a, b)
4969 }
4970 #[inline(always)]
4971 fn min_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
4972 crate::kernel!(
4973 #[inline(always)]
4974 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
4975 _mm_min_epu32(a.into(), b.into()).simd_into(token)
4976 }
4977 );
4978 kernel(self, a, b)
4979 }
4980 #[inline(always)]
4981 fn clamp_u32x4(self, a: u32x4<Self>, b: u32x4<Self>, c: u32x4<Self>) -> u32x4<Self> {
4982 debug_assert!(
4983 b.simd_le(c).all_true(),
4984 "clamp requires min <= max for all elements"
4985 );
4986 a.max(b).min(c)
4987 }
4988 #[inline(always)]
4989 fn simd_eq_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
4990 crate::kernel!(
4991 #[inline(always)]
4992 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> mask32x4<Sse4_2> {
4993 _mm_cmpeq_epi32(a.into(), b.into()).simd_into(token)
4994 }
4995 );
4996 kernel(self, a, b)
4997 }
4998 #[inline(always)]
4999 fn simd_lt_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
5000 crate::kernel!(
5001 #[inline(always)]
5002 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> mask32x4<Sse4_2> {
5003 {
5004 let sign_bit = _mm_set1_epi32(0x80000000u32.cast_signed());
5005 let lhs_signed = _mm_xor_si128(b.into(), sign_bit);
5006 let rhs_signed = _mm_xor_si128(a.into(), sign_bit);
5007 _mm_cmpgt_epi32(lhs_signed, rhs_signed)
5008 }
5009 .simd_into(token)
5010 }
5011 );
5012 kernel(self, a, b)
5013 }
5014 #[inline(always)]
5015 fn simd_le_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
5016 crate::kernel!(
5017 #[inline(always)]
5018 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> mask32x4<Sse4_2> {
5019 _mm_cmpeq_epi32(_mm_min_epu32(a.into(), b.into()), a.into()).simd_into(token)
5020 }
5021 );
5022 kernel(self, a, b)
5023 }
5024 #[inline(always)]
5025 fn zip_low_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5026 crate::kernel!(
5027 #[inline(always)]
5028 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
5029 _mm_unpacklo_epi32(a.into(), b.into()).simd_into(token)
5030 }
5031 );
5032 kernel(self, a, b)
5033 }
5034 #[inline(always)]
5035 fn zip_high_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5036 crate::kernel!(
5037 #[inline(always)]
5038 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
5039 _mm_unpackhi_epi32(a.into(), b.into()).simd_into(token)
5040 }
5041 );
5042 kernel(self, a, b)
5043 }
5044 #[inline(always)]
5045 fn unzip_low_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5046 crate::kernel!(
5047 #[inline(always)]
5048 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
5049 let t1 = _mm_shuffle_epi32::<0b11_01_10_00>(a.into());
5050 let t2 = _mm_shuffle_epi32::<0b11_01_10_00>(b.into());
5051 _mm_unpacklo_epi64(t1, t2).simd_into(token)
5052 }
5053 );
5054 kernel(self, a, b)
5055 }
5056 #[inline(always)]
5057 fn unzip_high_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5058 crate::kernel!(
5059 #[inline(always)]
5060 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u32x4<Sse4_2> {
5061 let t1 = _mm_shuffle_epi32::<0b11_01_10_00>(a.into());
5062 let t2 = _mm_shuffle_epi32::<0b11_01_10_00>(b.into());
5063 _mm_unpackhi_epi64(t1, t2).simd_into(token)
5064 }
5065 );
5066 kernel(self, a, b)
5067 }
5068 #[inline(always)]
5069 fn interleave_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u32x4<Self>, u32x4<Self>) {
5070 (self.zip_low_u32x4(a, b), self.zip_high_u32x4(a, b))
5071 }
5072 #[inline(always)]
5073 fn deinterleave_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u32x4<Self>, u32x4<Self>) {
5074 (self.unzip_low_u32x4(a, b), self.unzip_high_u32x4(a, b))
5075 }
5076 #[inline(always)]
5077 fn select_u32x4(self, a: mask32x4<Self>, b: u32x4<Self>, c: u32x4<Self>) -> u32x4<Self> {
5078 crate::kernel!(
5079 #[inline(always)]
5080 fn kernel(
5081 token: Sse4_2,
5082 a: mask32x4<Sse4_2>,
5083 b: u32x4<Sse4_2>,
5084 c: u32x4<Sse4_2>,
5085 ) -> u32x4<Sse4_2> {
5086 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
5087 }
5088 );
5089 kernel(self, a, b, c)
5090 }
5091 #[inline(always)]
5092 fn combine_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x8<Self> {
5093 u32x8 {
5094 val: crate::support::Aligned256([a.val.0, b.val.0]),
5095 simd: self,
5096 }
5097 }
5098 #[inline(always)]
5099 fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4<Self>; 4usize] {
5100 crate::kernel!(
5101 #[inline(always)]
5102 fn kernel(token: Sse4_2, src: &[u32; 16usize]) -> [u32x4<Sse4_2>; 4usize] {
5103 let (chunks, []) = src.as_chunks::<4usize>() else {
5104 unreachable!()
5105 };
5106 let v0: __m128i =
5107 crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]);
5108 let v1: __m128i =
5109 crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]);
5110 let v2: __m128i =
5111 crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]);
5112 let v3: __m128i =
5113 crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]);
5114 let tmp0 = _mm_unpacklo_epi32(v0, v1);
5115 let tmp1 = _mm_unpackhi_epi32(v0, v1);
5116 let tmp2 = _mm_unpacklo_epi32(v2, v3);
5117 let tmp3 = _mm_unpackhi_epi32(v2, v3);
5118 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
5119 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
5120 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
5121 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
5122 [
5123 out0.simd_into(token),
5124 out1.simd_into(token),
5125 out2.simd_into(token),
5126 out3.simd_into(token),
5127 ]
5128 }
5129 );
5130 kernel(self, src)
5131 }
5132 #[inline(always)]
5133 fn store_four_interleaved_u32x4(
5134 self,
5135 vectors: [u32x4<Self>; 4usize],
5136 dest: &mut [u32; 16usize],
5137 ) -> () {
5138 crate::kernel!(
5139 #[inline(always)]
5140 fn kernel(
5141 token: Sse4_2,
5142 vectors: [u32x4<Sse4_2>; 4usize],
5143 dest: &mut [u32; 16usize],
5144 ) -> () {
5145 let _ = token;
5146 let v0: __m128i = vectors[0].into();
5147 let v1: __m128i = vectors[1].into();
5148 let v2: __m128i = vectors[2].into();
5149 let v3: __m128i = vectors[3].into();
5150 let tmp0 = _mm_unpacklo_epi32(v0, v1);
5151 let tmp1 = _mm_unpackhi_epi32(v0, v1);
5152 let tmp2 = _mm_unpacklo_epi32(v2, v3);
5153 let tmp3 = _mm_unpackhi_epi32(v2, v3);
5154 let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
5155 let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
5156 let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
5157 let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
5158 let (chunks, []) = dest.as_chunks_mut::<4usize>() else {
5159 unreachable!()
5160 };
5161 crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>(
5162 out0,
5163 &mut chunks[0],
5164 );
5165 crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>(
5166 out1,
5167 &mut chunks[1],
5168 );
5169 crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>(
5170 out2,
5171 &mut chunks[2],
5172 );
5173 crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>(
5174 out3,
5175 &mut chunks[3],
5176 );
5177 }
5178 );
5179 kernel(self, vectors, dest);
5180 }
5181 #[inline(always)]
5182 fn widen_u32x4(self, a: u32x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
5183 crate::kernel!(
5184 #[inline(always)]
5185 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>) -> (u64x2<Sse4_2>, u64x2<Sse4_2>) {
5186 let raw = a.into();
5187 (
5188 _mm_cvtepu32_epi64(raw).simd_into(token),
5189 _mm_cvtepu32_epi64(_mm_srli_si128::<8>(raw)).simd_into(token),
5190 )
5191 }
5192 );
5193 kernel(self, a)
5194 }
5195 #[inline(always)]
5196 fn widening_mul_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
5197 let (a0, a1) = self.widen_u32x4(a);
5198 let (b0, b1) = self.widen_u32x4(b);
5199 (self.mul_u64x2(a0, b0), self.mul_u64x2(a1, b1))
5200 }
5201 #[inline(always)]
5202 fn widening_add_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
5203 let (a0, a1) = self.widen_u32x4(a);
5204 let (b0, b1) = self.widen_u32x4(b);
5205 (self.add_u64x2(a0, b0), self.add_u64x2(a1, b1))
5206 }
5207 #[inline(always)]
5208 fn pairwise_widening_add_u32x4(self, a: u32x4<Self>) -> u64x2<Self> {
5209 crate::kernel!(
5210 #[inline(always)]
5211 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>) -> u64x2<Sse4_2> {
5212 ({
5213 let a = a.into();
5214 _mm_add_epi64(
5215 _mm_and_si128(a, _mm_set1_epi64x(4294967295)),
5216 _mm_srli_epi64::<32>(a),
5217 )
5218 })
5219 .simd_into(token)
5220 }
5221 );
5222 kernel(self, a)
5223 }
5224 #[inline(always)]
5225 fn pairwise_dot_product_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u64x2<Self> {
5226 let (low, high) = self.widening_mul_u32x4(a, b);
5227 let (even, odd) = self.deinterleave_u64x2(low, high);
5228 self.add_u64x2(even, odd)
5229 }
5230 #[inline(always)]
5231 fn narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
5232 crate::kernel!(
5233 #[inline(always)]
5234 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u16x8<Sse4_2> {
5235 let mask = _mm_set1_epi32(0xffff);
5236 _mm_packus_epi32(_mm_and_si128(a.into(), mask), _mm_and_si128(b.into(), mask))
5237 .simd_into(token)
5238 }
5239 );
5240 kernel(self, a, b)
5241 }
5242 #[inline(always)]
5243 fn saturating_narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
5244 crate::kernel!(
5245 #[inline(always)]
5246 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u16x8<Sse4_2> {
5247 {
5248 let max = _mm_set1_epi32(i32::MAX);
5249 _mm_packus_epi32(_mm_min_epu32(a.into(), max), _mm_min_epu32(b.into(), max))
5250 }
5251 .simd_into(token)
5252 }
5253 );
5254 kernel(self, a, b)
5255 }
5256 #[inline(always)]
5257 fn relaxed_narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
5258 crate::kernel!(
5259 #[inline(always)]
5260 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>, b: u32x4<Sse4_2>) -> u16x8<Sse4_2> {
5261 debug_assert!(
5262 a.as_slice()
5263 .iter()
5264 .chain(b.as_slice())
5265 .all(|&value| { value >= u16::MIN as u32 && value <= u16::MAX as u32 }),
5266 "relaxed_narrow inputs must fit in the destination type",
5267 );
5268 _mm_packus_epi32(a.into(), b.into()).simd_into(token)
5269 }
5270 );
5271 kernel(self, a, b)
5272 }
5273 #[inline(always)]
5274 fn cvt_f32_u32x4(self, a: u32x4<Self>) -> f32x4<Self> {
5275 crate::kernel!(
5276 #[inline(always)]
5277 fn kernel(token: Sse4_2, a: u32x4<Sse4_2>) -> f32x4<Sse4_2> {
5278 let a = a.into();
5279 let lo = _mm_blend_epi16::<0xAA>(a, _mm_set1_epi32(0x4B000000));
5280 let hi =
5281 _mm_blend_epi16::<0xAA>(_mm_srli_epi32::<16>(a), _mm_set1_epi32(0x53000000));
5282 let fhi = _mm_sub_ps(
5283 _mm_castsi128_ps(hi),
5284 _mm_set1_ps(f32::from_bits(0x53000080)),
5285 );
5286 let result = _mm_add_ps(_mm_castsi128_ps(lo), fhi);
5287 result.simd_into(token)
5288 }
5289 );
5290 kernel(self, a)
5291 }
5292 #[inline(always)]
5293 fn splat_mask32x4(self, val: bool) -> mask32x4<Self> {
5294 crate::kernel!(
5295 #[inline(always)]
5296 fn kernel(token: Sse4_2, val: bool) -> mask32x4<Sse4_2> {
5297 let val: i32 = if val { !0 } else { 0 };
5298 _mm_set1_epi32(val).simd_into(token)
5299 }
5300 );
5301 kernel(self, val)
5302 }
5303 #[inline(always)]
5304 fn from_bitmask_mask32x4(self, bits: u64) -> mask32x4<Self> {
5305 crate::kernel!(
5306 #[inline(always)]
5307 fn kernel(token: Sse4_2, bits: u64) -> mask32x4<Sse4_2> {
5308 {
5309 let bit_lanes = _mm_set1_epi32(bits as i32);
5310 let bit_mask = _mm_setr_epi32(1, 2, 4, 8);
5311 _mm_cmpeq_epi32(_mm_and_si128(bit_lanes, bit_mask), bit_mask)
5312 }
5313 .simd_into(token)
5314 }
5315 );
5316 kernel(self, bits)
5317 }
5318 #[inline(always)]
5319 fn to_bitmask_mask32x4(self, a: mask32x4<Self>) -> u64 {
5320 crate::kernel!(
5321 #[inline(always)]
5322 fn kernel(token: Sse4_2, a: mask32x4<Sse4_2>) -> u64 {
5323 _mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 as u64
5324 }
5325 );
5326 kernel(self, a)
5327 }
5328 #[inline(always)]
5329 fn from_vector_mask32x4(self, vector: i32x4<Self>) -> mask32x4<Self> {
5330 mask32x4 {
5331 val: vector.val,
5332 simd: self,
5333 }
5334 }
5335 #[inline(always)]
5336 fn to_vector_mask32x4(self, a: mask32x4<Self>) -> i32x4<Self> {
5337 i32x4 {
5338 val: a.val,
5339 simd: self,
5340 }
5341 }
5342 #[inline(always)]
5343 fn set_mask32x4(self, a: &mut mask32x4<Self>, index: usize, value: bool) -> () {
5344 assert!(
5345 index < 4usize,
5346 "mask lane index {index} is out of bounds for {} lanes",
5347 4usize
5348 );
5349 let mut lanes: [i32; 4usize] = (*a).into();
5350 lanes[index] = if value { !0 } else { 0 };
5351 *a = lanes.simd_into(self);
5352 }
5353 #[inline(always)]
5354 fn rotate_elements_left_mask32x4<const OFFSET: usize>(
5355 self,
5356 a: mask32x4<Self>,
5357 ) -> mask32x4<Self> {
5358 let int = i32x4 {
5359 val: crate::transmute::checked_transmute_copy(&a.val),
5360 simd: self,
5361 };
5362 let rotated = int.rotate_elements_left::<OFFSET>();
5363 mask32x4 {
5364 val: crate::transmute::checked_transmute_copy(&rotated.val),
5365 simd: self,
5366 }
5367 }
5368 #[inline(always)]
5369 fn rotate_elements_right_mask32x4<const OFFSET: usize>(
5370 self,
5371 a: mask32x4<Self>,
5372 ) -> mask32x4<Self> {
5373 let int = i32x4 {
5374 val: crate::transmute::checked_transmute_copy(&a.val),
5375 simd: self,
5376 };
5377 let rotated = int.rotate_elements_right::<OFFSET>();
5378 mask32x4 {
5379 val: crate::transmute::checked_transmute_copy(&rotated.val),
5380 simd: self,
5381 }
5382 }
5383 #[inline(always)]
5384 fn and_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
5385 crate::kernel!(
5386 #[inline(always)]
5387 fn kernel(token: Sse4_2, a: mask32x4<Sse4_2>, b: mask32x4<Sse4_2>) -> mask32x4<Sse4_2> {
5388 _mm_and_si128(a.into(), b.into()).simd_into(token)
5389 }
5390 );
5391 kernel(self, a, b)
5392 }
5393 #[inline(always)]
5394 fn or_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
5395 crate::kernel!(
5396 #[inline(always)]
5397 fn kernel(token: Sse4_2, a: mask32x4<Sse4_2>, b: mask32x4<Sse4_2>) -> mask32x4<Sse4_2> {
5398 _mm_or_si128(a.into(), b.into()).simd_into(token)
5399 }
5400 );
5401 kernel(self, a, b)
5402 }
5403 #[inline(always)]
5404 fn xor_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
5405 crate::kernel!(
5406 #[inline(always)]
5407 fn kernel(token: Sse4_2, a: mask32x4<Sse4_2>, b: mask32x4<Sse4_2>) -> mask32x4<Sse4_2> {
5408 _mm_xor_si128(a.into(), b.into()).simd_into(token)
5409 }
5410 );
5411 kernel(self, a, b)
5412 }
5413 #[inline(always)]
5414 fn not_mask32x4(self, a: mask32x4<Self>) -> mask32x4<Self> {
5415 self.xor_mask32x4(a, self.splat_mask32x4(true))
5416 }
5417 #[inline(always)]
5418 fn reverse_mask32x4(self, a: mask32x4<Self>) -> mask32x4<Self> {
5419 let lanes = i32x4 {
5420 val: crate::transmute::checked_transmute_copy(&a.val),
5421 simd: self,
5422 };
5423 let reversed = self.reverse_i32x4(lanes);
5424 mask32x4 {
5425 val: crate::transmute::checked_transmute_copy(&reversed.val),
5426 simd: self,
5427 }
5428 }
5429 #[inline(always)]
5430 fn select_mask32x4(
5431 self,
5432 a: mask32x4<Self>,
5433 b: mask32x4<Self>,
5434 c: mask32x4<Self>,
5435 ) -> mask32x4<Self> {
5436 crate::kernel!(
5437 #[inline(always)]
5438 fn kernel(
5439 token: Sse4_2,
5440 a: mask32x4<Sse4_2>,
5441 b: mask32x4<Sse4_2>,
5442 c: mask32x4<Sse4_2>,
5443 ) -> mask32x4<Sse4_2> {
5444 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
5445 }
5446 );
5447 kernel(self, a, b, c)
5448 }
5449 #[inline(always)]
5450 fn simd_eq_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
5451 crate::kernel!(
5452 #[inline(always)]
5453 fn kernel(token: Sse4_2, a: mask32x4<Sse4_2>, b: mask32x4<Sse4_2>) -> mask32x4<Sse4_2> {
5454 _mm_cmpeq_epi32(a.into(), b.into()).simd_into(token)
5455 }
5456 );
5457 kernel(self, a, b)
5458 }
5459 #[inline(always)]
5460 fn any_true_mask32x4(self, a: mask32x4<Self>) -> bool {
5461 crate::kernel!(
5462 #[inline(always)]
5463 fn kernel(token: Sse4_2, a: mask32x4<Sse4_2>) -> bool {
5464 _mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 != 0
5465 }
5466 );
5467 kernel(self, a)
5468 }
5469 #[inline(always)]
5470 fn all_true_mask32x4(self, a: mask32x4<Self>) -> bool {
5471 crate::kernel!(
5472 #[inline(always)]
5473 fn kernel(token: Sse4_2, a: mask32x4<Sse4_2>) -> bool {
5474 _mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 == 0b1111
5475 }
5476 );
5477 kernel(self, a)
5478 }
5479 #[inline(always)]
5480 fn any_false_mask32x4(self, a: mask32x4<Self>) -> bool {
5481 crate::kernel!(
5482 #[inline(always)]
5483 fn kernel(token: Sse4_2, a: mask32x4<Sse4_2>) -> bool {
5484 _mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 != 0b1111
5485 }
5486 );
5487 kernel(self, a)
5488 }
5489 #[inline(always)]
5490 fn all_false_mask32x4(self, a: mask32x4<Self>) -> bool {
5491 crate::kernel!(
5492 #[inline(always)]
5493 fn kernel(token: Sse4_2, a: mask32x4<Sse4_2>) -> bool {
5494 _mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 == 0
5495 }
5496 );
5497 kernel(self, a)
5498 }
5499 #[inline(always)]
5500 fn combine_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x8<Self> {
5501 mask32x8 {
5502 val: crate::support::Aligned256([a.val.0, b.val.0]),
5503 simd: self,
5504 }
5505 }
5506 #[inline(always)]
5507 fn widen_mask32x4(self, a: mask32x4<Self>) -> (mask64x2<Self>, mask64x2<Self>) {
5508 crate::kernel!(
5509 #[inline(always)]
5510 fn kernel(token: Sse4_2, a: mask32x4<Sse4_2>) -> (mask64x2<Sse4_2>, mask64x2<Sse4_2>) {
5511 let raw = a.into();
5512 (
5513 _mm_unpacklo_epi32(raw, raw).simd_into(token),
5514 _mm_unpackhi_epi32(raw, raw).simd_into(token),
5515 )
5516 }
5517 );
5518 kernel(self, a)
5519 }
5520 #[inline(always)]
5521 fn narrow_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask16x8<Self> {
5522 crate::kernel!(
5523 #[inline(always)]
5524 fn kernel(token: Sse4_2, a: mask32x4<Sse4_2>, b: mask32x4<Sse4_2>) -> mask16x8<Sse4_2> {
5525 _mm_packs_epi32(a.into(), b.into()).simd_into(token)
5526 }
5527 );
5528 kernel(self, a, b)
5529 }
5530 #[inline(always)]
5531 fn abs_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5532 crate::kernel!(
5533 #[inline(always)]
5534 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
5535 _mm_andnot_pd(_mm_set1_pd(-0.0), a.into()).simd_into(token)
5536 }
5537 );
5538 kernel(self, a)
5539 }
5540 #[inline(always)]
5541 fn splat_f64x2(self, val: f64) -> f64x2<Self> {
5542 crate::kernel!(
5543 #[inline(always)]
5544 fn kernel(token: Sse4_2, val: f64) -> f64x2<Sse4_2> {
5545 _mm_set1_pd(val).simd_into(token)
5546 }
5547 );
5548 kernel(self, val)
5549 }
5550 #[inline(always)]
5551 fn reverse_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5552 let indices: u8x16<Self> =
5553 [8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7].simd_into(self);
5554 self.swizzle_dyn_f64x2(a, indices)
5555 }
5556 #[inline(always)]
5557 fn slide_f64x2<const SHIFT: usize>(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5558 if SHIFT >= 2usize {
5559 return b;
5560 }
5561 let result = dyn_alignr_128(
5562 self,
5563 Bytes::to_bytes(b).val.0,
5564 Bytes::to_bytes(a).val.0,
5565 SHIFT * 8usize,
5566 );
5567 Bytes::from_bytes(u8x16 {
5568 val: crate::support::Aligned128(result),
5569 simd: self,
5570 })
5571 }
5572 #[inline(always)]
5573 fn compress_f64x2(self, values: f64x2<Self>, mask: mask64x2<Self>) -> f64x2<Self> {
5574 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
5575 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0);
5576 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
5577 }
5578 #[inline(always)]
5579 fn compress_merge_f64x2(
5580 self,
5581 values: f64x2<Self>,
5582 mask: mask64x2<Self>,
5583 merge: f64x2<Self>,
5584 ) -> f64x2<Self> {
5585 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
5586 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0);
5587 let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control);
5588 let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0));
5589 Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed))
5590 }
5591 #[inline(always)]
5592 fn expand_f64x2(self, values: f64x2<Self>, mask: mask64x2<Self>) -> f64x2<Self> {
5593 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
5594 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0);
5595 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
5596 }
5597 #[inline(always)]
5598 fn expand_merge_f64x2(
5599 self,
5600 values: f64x2<Self>,
5601 mask: mask64x2<Self>,
5602 merge: f64x2<Self>,
5603 ) -> f64x2<Self> {
5604 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
5605 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0);
5606 let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control));
5607 self.select_f64x2(mask, expanded, merge)
5608 }
5609 #[inline(always)]
5610 fn neg_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5611 crate::kernel!(
5612 #[inline(always)]
5613 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
5614 _mm_xor_pd(a.into(), _mm_set1_pd(-0.0)).simd_into(token)
5615 }
5616 );
5617 kernel(self, a)
5618 }
5619 #[inline(always)]
5620 fn sqrt_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5621 crate::kernel!(
5622 #[inline(always)]
5623 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
5624 _mm_sqrt_pd(a.into()).simd_into(token)
5625 }
5626 );
5627 kernel(self, a)
5628 }
5629 #[inline(always)]
5630 fn approximate_recip_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5631 1.0 / a
5632 }
5633 #[inline(always)]
5634 fn add_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5635 crate::kernel!(
5636 #[inline(always)]
5637 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
5638 _mm_add_pd(a.into(), b.into()).simd_into(token)
5639 }
5640 );
5641 kernel(self, a, b)
5642 }
5643 #[inline(always)]
5644 fn sub_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5645 crate::kernel!(
5646 #[inline(always)]
5647 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
5648 _mm_sub_pd(a.into(), b.into()).simd_into(token)
5649 }
5650 );
5651 kernel(self, a, b)
5652 }
5653 #[inline(always)]
5654 fn mul_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5655 crate::kernel!(
5656 #[inline(always)]
5657 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
5658 _mm_mul_pd(a.into(), b.into()).simd_into(token)
5659 }
5660 );
5661 kernel(self, a, b)
5662 }
5663 #[inline(always)]
5664 fn div_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5665 crate::kernel!(
5666 #[inline(always)]
5667 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
5668 _mm_div_pd(a.into(), b.into()).simd_into(token)
5669 }
5670 );
5671 kernel(self, a, b)
5672 }
5673 #[inline(always)]
5674 fn copysign_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5675 crate::kernel!(
5676 #[inline(always)]
5677 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
5678 let mask = _mm_set1_pd(-0.0);
5679 _mm_or_pd(_mm_and_pd(mask, b.into()), _mm_andnot_pd(mask, a.into()))
5680 .simd_into(token)
5681 }
5682 );
5683 kernel(self, a, b)
5684 }
5685 #[inline(always)]
5686 fn reduce_max_f64x2(self, a: f64x2<Self>) -> f64 {
5687 crate::kernel!(
5688 #[inline(always)]
5689 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f64 {
5690 let reduced: __m128d = a.into();
5691 let shifted = _mm_castsi128_pd(_mm_srli_si128::<8>(_mm_castpd_si128(reduced)));
5692 let reduced = { _mm_max_pd(reduced, shifted) };
5693 _mm_cvtsd_f64(reduced)
5694 }
5695 );
5696 kernel(self, a)
5697 }
5698 #[inline(always)]
5699 fn reduce_min_f64x2(self, a: f64x2<Self>) -> f64 {
5700 crate::kernel!(
5701 #[inline(always)]
5702 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f64 {
5703 let reduced: __m128d = a.into();
5704 let shifted = _mm_castsi128_pd(_mm_srli_si128::<8>(_mm_castpd_si128(reduced)));
5705 let reduced = { _mm_min_pd(reduced, shifted) };
5706 _mm_cvtsd_f64(reduced)
5707 }
5708 );
5709 kernel(self, a)
5710 }
5711 #[inline(always)]
5712 fn reduce_max_precise_f64x2(self, a: f64x2<Self>) -> f64 {
5713 crate::kernel!(
5714 #[inline(always)]
5715 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f64 {
5716 let reduced: __m128d = a.into();
5717 let shifted = _mm_castsi128_pd(_mm_srli_si128::<8>(_mm_castpd_si128(reduced)));
5718 let reduced = {
5719 let intermediate = _mm_max_pd(reduced, shifted);
5720 let b_is_nan = _mm_cmpunord_pd(shifted, shifted);
5721 _mm_blendv_pd(intermediate, reduced, b_is_nan)
5722 };
5723 _mm_cvtsd_f64(reduced)
5724 }
5725 );
5726 kernel(self, a)
5727 }
5728 #[inline(always)]
5729 fn reduce_min_precise_f64x2(self, a: f64x2<Self>) -> f64 {
5730 crate::kernel!(
5731 #[inline(always)]
5732 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f64 {
5733 let reduced: __m128d = a.into();
5734 let shifted = _mm_castsi128_pd(_mm_srli_si128::<8>(_mm_castpd_si128(reduced)));
5735 let reduced = {
5736 let intermediate = _mm_min_pd(reduced, shifted);
5737 let b_is_nan = _mm_cmpunord_pd(shifted, shifted);
5738 _mm_blendv_pd(intermediate, reduced, b_is_nan)
5739 };
5740 _mm_cvtsd_f64(reduced)
5741 }
5742 );
5743 kernel(self, a)
5744 }
5745 #[inline(always)]
5746 fn reduce_sum_f64x2(self, a: f64x2<Self>) -> f64 {
5747 crate::kernel!(
5748 #[inline(always)]
5749 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f64 {
5750 let a: __m128d = a.into();
5751 _mm_cvtsd_f64(_mm_add_sd(a, _mm_unpackhi_pd(a, a)))
5752 }
5753 );
5754 kernel(self, a)
5755 }
5756 #[inline(always)]
5757 fn reduce_product_f64x2(self, a: f64x2<Self>) -> f64 {
5758 crate::kernel!(
5759 #[inline(always)]
5760 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f64 {
5761 let a: __m128d = a.into();
5762 _mm_cvtsd_f64(_mm_mul_sd(a, _mm_unpackhi_pd(a, a)))
5763 }
5764 );
5765 kernel(self, a)
5766 }
5767 #[inline(always)]
5768 fn max_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5769 crate::kernel!(
5770 #[inline(always)]
5771 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
5772 _mm_max_pd(a.into(), b.into()).simd_into(token)
5773 }
5774 );
5775 kernel(self, a, b)
5776 }
5777 #[inline(always)]
5778 fn min_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5779 crate::kernel!(
5780 #[inline(always)]
5781 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
5782 _mm_min_pd(a.into(), b.into()).simd_into(token)
5783 }
5784 );
5785 kernel(self, a, b)
5786 }
5787 #[inline(always)]
5788 fn max_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5789 crate::kernel!(
5790 #[inline(always)]
5791 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
5792 let intermediate = _mm_max_pd(a.into(), b.into());
5793 let b_is_nan = _mm_cmpunord_pd(b.into(), b.into());
5794 _mm_blendv_pd(intermediate, a.into(), b_is_nan).simd_into(token)
5795 }
5796 );
5797 kernel(self, a, b)
5798 }
5799 #[inline(always)]
5800 fn min_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5801 crate::kernel!(
5802 #[inline(always)]
5803 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
5804 let intermediate = _mm_min_pd(a.into(), b.into());
5805 let b_is_nan = _mm_cmpunord_pd(b.into(), b.into());
5806 _mm_blendv_pd(intermediate, a.into(), b_is_nan).simd_into(token)
5807 }
5808 );
5809 kernel(self, a, b)
5810 }
5811 #[inline(always)]
5812 fn clamp_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
5813 crate::kernel!(
5814 #[inline(always)]
5815 fn kernel(
5816 token: Sse4_2,
5817 a: f64x2<Sse4_2>,
5818 b: f64x2<Sse4_2>,
5819 c: f64x2<Sse4_2>,
5820 ) -> f64x2<Sse4_2> {
5821 debug_assert!(
5822 b.simd_le(c).all_true(),
5823 "clamp requires min <= max for all elements"
5824 );
5825 _mm_min_pd(c.into(), _mm_max_pd(b.into(), a.into())).simd_into(token)
5826 }
5827 );
5828 kernel(self, a, b, c)
5829 }
5830 #[inline(always)]
5831 fn simd_eq_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
5832 crate::kernel!(
5833 #[inline(always)]
5834 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> mask64x2<Sse4_2> {
5835 _mm_castpd_si128(_mm_cmpeq_pd(a.into(), b.into())).simd_into(token)
5836 }
5837 );
5838 kernel(self, a, b)
5839 }
5840 #[inline(always)]
5841 fn simd_ne_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
5842 crate::kernel!(
5843 #[inline(always)]
5844 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> mask64x2<Sse4_2> {
5845 _mm_castpd_si128(_mm_cmpneq_pd(a.into(), b.into())).simd_into(token)
5846 }
5847 );
5848 kernel(self, a, b)
5849 }
5850 #[inline(always)]
5851 fn simd_lt_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
5852 crate::kernel!(
5853 #[inline(always)]
5854 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> mask64x2<Sse4_2> {
5855 _mm_castpd_si128(_mm_cmplt_pd(a.into(), b.into())).simd_into(token)
5856 }
5857 );
5858 kernel(self, a, b)
5859 }
5860 #[inline(always)]
5861 fn simd_le_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
5862 crate::kernel!(
5863 #[inline(always)]
5864 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> mask64x2<Sse4_2> {
5865 _mm_castpd_si128(_mm_cmple_pd(a.into(), b.into())).simd_into(token)
5866 }
5867 );
5868 kernel(self, a, b)
5869 }
5870 #[inline(always)]
5871 fn zip_low_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5872 crate::kernel!(
5873 #[inline(always)]
5874 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
5875 _mm_unpacklo_pd(a.into(), b.into()).simd_into(token)
5876 }
5877 );
5878 kernel(self, a, b)
5879 }
5880 #[inline(always)]
5881 fn zip_high_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5882 crate::kernel!(
5883 #[inline(always)]
5884 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
5885 _mm_unpackhi_pd(a.into(), b.into()).simd_into(token)
5886 }
5887 );
5888 kernel(self, a, b)
5889 }
5890 #[inline(always)]
5891 fn unzip_low_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5892 crate::kernel!(
5893 #[inline(always)]
5894 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
5895 _mm_shuffle_pd::<0b00>(a.into(), b.into()).simd_into(token)
5896 }
5897 );
5898 kernel(self, a, b)
5899 }
5900 #[inline(always)]
5901 fn unzip_high_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
5902 crate::kernel!(
5903 #[inline(always)]
5904 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
5905 _mm_shuffle_pd::<0b11>(a.into(), b.into()).simd_into(token)
5906 }
5907 );
5908 kernel(self, a, b)
5909 }
5910 #[inline(always)]
5911 fn interleave_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> (f64x2<Self>, f64x2<Self>) {
5912 (self.zip_low_f64x2(a, b), self.zip_high_f64x2(a, b))
5913 }
5914 #[inline(always)]
5915 fn deinterleave_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> (f64x2<Self>, f64x2<Self>) {
5916 (self.unzip_low_f64x2(a, b), self.unzip_high_f64x2(a, b))
5917 }
5918 #[inline(always)]
5919 fn mul_add_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
5920 a * b + c
5921 }
5922 #[inline(always)]
5923 fn mul_add_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
5924 crate::math::mul_add_precise_f64x2(self, a, b, c)
5925 }
5926 #[inline(always)]
5927 fn mul_sub_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
5928 a * b - c
5929 }
5930 #[inline(always)]
5931 fn mul_sub_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
5932 self.mul_add_precise_f64x2(a, b, -c)
5933 }
5934 #[inline(always)]
5935 fn floor_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5936 crate::kernel!(
5937 #[inline(always)]
5938 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
5939 _mm_round_pd::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
5940 .simd_into(token)
5941 }
5942 );
5943 kernel(self, a)
5944 }
5945 #[inline(always)]
5946 fn ceil_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5947 crate::kernel!(
5948 #[inline(always)]
5949 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
5950 _mm_round_pd::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
5951 .simd_into(token)
5952 }
5953 );
5954 kernel(self, a)
5955 }
5956 #[inline(always)]
5957 fn round_ties_even_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5958 crate::kernel!(
5959 #[inline(always)]
5960 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
5961 _mm_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
5962 .simd_into(token)
5963 }
5964 );
5965 kernel(self, a)
5966 }
5967 #[inline(always)]
5968 fn round_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5969 let bias = self.splat_f64x2(const { f64::next_down(0.5) });
5970 (a + self.copysign_f64x2(bias, a)).trunc()
5971 }
5972 #[inline(always)]
5973 fn fract_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5974 a - self.trunc_f64x2(a)
5975 }
5976 #[inline(always)]
5977 fn trunc_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5978 crate::kernel!(
5979 #[inline(always)]
5980 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> f64x2<Sse4_2> {
5981 _mm_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
5982 .simd_into(token)
5983 }
5984 );
5985 kernel(self, a)
5986 }
5987 #[inline(always)]
5988 fn to_degrees_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5989 a * const { f64::to_degrees(1.0) }
5990 }
5991 #[inline(always)]
5992 fn to_radians_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
5993 a * const { f64::to_radians(1.0) }
5994 }
5995 #[inline(always)]
5996 fn is_nan_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
5997 a.simd_ne(a)
5998 }
5999 #[inline(always)]
6000 fn is_infinite_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6001 a.abs().simd_eq(f64::INFINITY)
6002 }
6003 #[inline(always)]
6004 fn is_finite_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6005 a.abs().simd_lt(f64::INFINITY)
6006 }
6007 #[inline(always)]
6008 fn is_subnormal_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6009 let i: i64x2<Self> = a.bitcast();
6010 (i & 0x7FF0_0000_0000_0000_i64).simd_eq(0) & (i & 0x000F_FFFF_FFFF_FFFF_i64).simd_ne(0)
6011 }
6012 #[inline(always)]
6013 fn is_normal_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6014 let i: i64x2<Self> = a.bitcast();
6015 let exp = i & 0x7FF0_0000_0000_0000_i64;
6016 exp.simd_ne(0) & exp.simd_ne(0x7FF0_0000_0000_0000_i64)
6017 }
6018 #[inline(always)]
6019 fn is_sign_positive_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6020 let i: i64x2<Self> = a.bitcast();
6021 (i & i64::MIN).simd_eq(0)
6022 }
6023 #[inline(always)]
6024 fn is_sign_negative_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6025 !a.is_sign_positive()
6026 }
6027 #[inline(always)]
6028 fn select_f64x2(self, a: mask64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
6029 crate::kernel!(
6030 #[inline(always)]
6031 fn kernel(
6032 token: Sse4_2,
6033 a: mask64x2<Sse4_2>,
6034 b: f64x2<Sse4_2>,
6035 c: f64x2<Sse4_2>,
6036 ) -> f64x2<Sse4_2> {
6037 _mm_blendv_pd(c.into(), b.into(), _mm_castsi128_pd(a.into())).simd_into(token)
6038 }
6039 );
6040 kernel(self, a, b, c)
6041 }
6042 #[inline(always)]
6043 fn combine_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x4<Self> {
6044 f64x4 {
6045 val: crate::support::Aligned256([a.val.0, b.val.0]),
6046 simd: self,
6047 }
6048 }
6049 #[inline(always)]
6050 fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2<Self>; 4usize] {
6051 crate::kernel!(
6052 #[inline(always)]
6053 fn kernel(token: Sse4_2, src: &[f64; 8usize]) -> [f64x2<Sse4_2>; 4usize] {
6054 let (chunks, []) = src.as_chunks::<2usize>() else {
6055 unreachable!()
6056 };
6057 let v0: __m128d =
6058 crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[0]);
6059 let v1: __m128d =
6060 crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[1]);
6061 let v2: __m128d =
6062 crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[2]);
6063 let v3: __m128d =
6064 crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[3]);
6065 let out0 = _mm_unpacklo_pd(v0, v2);
6066 let out1 = _mm_unpackhi_pd(v0, v2);
6067 let out2 = _mm_unpacklo_pd(v1, v3);
6068 let out3 = _mm_unpackhi_pd(v1, v3);
6069 [
6070 out0.simd_into(token),
6071 out1.simd_into(token),
6072 out2.simd_into(token),
6073 out3.simd_into(token),
6074 ]
6075 }
6076 );
6077 kernel(self, src)
6078 }
6079 #[inline(always)]
6080 fn store_four_interleaved_f64x2(
6081 self,
6082 vectors: [f64x2<Self>; 4usize],
6083 dest: &mut [f64; 8usize],
6084 ) -> () {
6085 crate::kernel!(
6086 #[inline(always)]
6087 fn kernel(
6088 token: Sse4_2,
6089 vectors: [f64x2<Sse4_2>; 4usize],
6090 dest: &mut [f64; 8usize],
6091 ) -> () {
6092 let _ = token;
6093 let v0: __m128d = vectors[0].into();
6094 let v1: __m128d = vectors[1].into();
6095 let v2: __m128d = vectors[2].into();
6096 let v3: __m128d = vectors[3].into();
6097 let out0 = _mm_unpacklo_pd(v0, v1);
6098 let out1 = _mm_unpacklo_pd(v2, v3);
6099 let out2 = _mm_unpackhi_pd(v0, v1);
6100 let out3 = _mm_unpackhi_pd(v2, v3);
6101 let (chunks, []) = dest.as_chunks_mut::<2usize>() else {
6102 unreachable!()
6103 };
6104 crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>(
6105 out0,
6106 &mut chunks[0],
6107 );
6108 crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>(
6109 out1,
6110 &mut chunks[1],
6111 );
6112 crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>(
6113 out2,
6114 &mut chunks[2],
6115 );
6116 crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>(
6117 out3,
6118 &mut chunks[3],
6119 );
6120 }
6121 );
6122 kernel(self, vectors, dest);
6123 }
6124 #[inline(always)]
6125 fn narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
6126 crate::kernel!(
6127 #[inline(always)]
6128 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>, b: f64x2<Sse4_2>) -> f32x4<Sse4_2> {
6129 let low = _mm_cvtpd_ps(a.into());
6130 let high = _mm_cvtpd_ps(b.into());
6131 _mm_movelh_ps(low, high).simd_into(token)
6132 }
6133 );
6134 kernel(self, a, b)
6135 }
6136 #[inline(always)]
6137 fn saturating_narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
6138 self.narrow_f64x2(a, b)
6139 }
6140 #[inline(always)]
6141 fn relaxed_narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
6142 self.narrow_f64x2(a, b)
6143 }
6144 #[inline(always)]
6145 fn cvt_u64_f64x2(self, a: f64x2<Self>) -> u64x2<Self> {
6146 [a[0usize] as u64, a[1usize] as u64].simd_into(self)
6147 }
6148 #[inline(always)]
6149 fn cvt_u64_precise_f64x2(self, a: f64x2<Self>) -> u64x2<Self> {
6150 [a[0usize] as u64, a[1usize] as u64].simd_into(self)
6151 }
6152 #[inline(always)]
6153 #[cfg(target_arch = "x86_64")]
6154 fn cvt_i64_f64x2(self, a: f64x2<Self>) -> i64x2<Self> {
6155 crate::kernel!(
6156 #[inline(always)]
6157 fn kernel(token: Sse4_2, a: f64x2<Sse4_2>) -> i64x2<Sse4_2> {
6158 let a = a.into();
6159 let low = _mm_cvttsd_si64(a);
6160 let high = _mm_cvttsd_si64(_mm_unpackhi_pd(a, a));
6161 _mm_set_epi64x(high, low).simd_into(token)
6162 }
6163 );
6164 kernel(self, a)
6165 }
6166 #[cfg(target_arch = "x86")]
6167 fn cvt_i64_f64x2(self, a: f64x2<Self>) -> i64x2<Self> {
6168 [a[0usize] as i64, a[1usize] as i64].simd_into(self)
6169 }
6170 #[inline(always)]
6171 fn cvt_i64_precise_f64x2(self, a: f64x2<Self>) -> i64x2<Self> {
6172 [a[0usize] as i64, a[1usize] as i64].simd_into(self)
6173 }
6174 #[inline(always)]
6175 fn abs_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6176 crate::kernel!(
6177 #[inline(always)]
6178 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>) -> i64x2<Sse4_2> {
6179 let a = a.into();
6180 let mask = _mm_cmpgt_epi64(_mm_setzero_si128(), a);
6181 _mm_sub_epi64(_mm_xor_si128(a, mask), mask).simd_into(token)
6182 }
6183 );
6184 kernel(self, a)
6185 }
6186 #[inline(always)]
6187 fn splat_i64x2(self, val: i64) -> i64x2<Self> {
6188 crate::kernel!(
6189 #[inline(always)]
6190 fn kernel(token: Sse4_2, val: i64) -> i64x2<Sse4_2> {
6191 _mm_set1_epi64x(val).simd_into(token)
6192 }
6193 );
6194 kernel(self, val)
6195 }
6196 #[inline(always)]
6197 fn reverse_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6198 let indices: u8x16<Self> =
6199 [8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7].simd_into(self);
6200 self.swizzle_dyn_i64x2(a, indices)
6201 }
6202 #[inline(always)]
6203 fn slide_i64x2<const SHIFT: usize>(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6204 if SHIFT >= 2usize {
6205 return b;
6206 }
6207 let result = dyn_alignr_128(
6208 self,
6209 Bytes::to_bytes(b).val.0,
6210 Bytes::to_bytes(a).val.0,
6211 SHIFT * 8usize,
6212 );
6213 Bytes::from_bytes(u8x16 {
6214 val: crate::support::Aligned128(result),
6215 simd: self,
6216 })
6217 }
6218 #[inline(always)]
6219 fn compress_i64x2(self, values: i64x2<Self>, mask: mask64x2<Self>) -> i64x2<Self> {
6220 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
6221 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0);
6222 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
6223 }
6224 #[inline(always)]
6225 fn compress_merge_i64x2(
6226 self,
6227 values: i64x2<Self>,
6228 mask: mask64x2<Self>,
6229 merge: i64x2<Self>,
6230 ) -> i64x2<Self> {
6231 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
6232 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0);
6233 let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control);
6234 let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0));
6235 Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed))
6236 }
6237 #[inline(always)]
6238 fn expand_i64x2(self, values: i64x2<Self>, mask: mask64x2<Self>) -> i64x2<Self> {
6239 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
6240 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0);
6241 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
6242 }
6243 #[inline(always)]
6244 fn expand_merge_i64x2(
6245 self,
6246 values: i64x2<Self>,
6247 mask: mask64x2<Self>,
6248 merge: i64x2<Self>,
6249 ) -> i64x2<Self> {
6250 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
6251 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0);
6252 let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control));
6253 self.select_i64x2(mask, expanded, merge)
6254 }
6255 #[inline(always)]
6256 fn count_ones_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6257 crate::kernel!(
6258 #[inline(always)]
6259 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>) -> i64x2<Sse4_2> {
6260 let value = a.into();
6261 let nibble_mask = _mm_set1_epi8(0x0f);
6262 let lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4);
6263 let low = _mm_and_si128(value, nibble_mask);
6264 let high = _mm_and_si128(_mm_srli_epi16::<4>(value), nibble_mask);
6265 let byte_counts = _mm_add_epi8(
6266 _mm_shuffle_epi8(lookup, low),
6267 _mm_shuffle_epi8(lookup, high),
6268 );
6269 _mm_sad_epu8(byte_counts, _mm_setzero_si128()).simd_into(token)
6270 }
6271 );
6272 kernel(self, a)
6273 }
6274 #[inline(always)]
6275 fn count_zeros_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6276 self.count_ones_i64x2(self.not_i64x2(a))
6277 }
6278 #[inline(always)]
6279 fn add_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6280 crate::kernel!(
6281 #[inline(always)]
6282 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, b: i64x2<Sse4_2>) -> i64x2<Sse4_2> {
6283 _mm_add_epi64(a.into(), b.into()).simd_into(token)
6284 }
6285 );
6286 kernel(self, a, b)
6287 }
6288 #[inline(always)]
6289 fn saturating_add_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6290 crate::kernel!(
6291 #[inline(always)]
6292 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, b: i64x2<Sse4_2>) -> i64x2<Sse4_2> {
6293 let a = a.into();
6294 let b = b.into();
6295 let wrapped = _mm_add_epi64(a, b);
6296 let overflow = _mm_xor_si128(_mm_cmpgt_epi64(a, wrapped), b);
6297 let a_sign = _mm_srai_epi32::<31>(_mm_shuffle_epi32::<0xf5>(a));
6298 let bound = _mm_xor_si128(a_sign, _mm_set1_epi64x(i64::MAX));
6299 let result = _mm_blendv_pd(
6300 _mm_castsi128_pd(wrapped),
6301 _mm_castsi128_pd(bound),
6302 _mm_castsi128_pd(overflow),
6303 );
6304 _mm_castpd_si128(result).simd_into(token)
6305 }
6306 );
6307 kernel(self, a, b)
6308 }
6309 #[inline(always)]
6310 fn sub_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6311 crate::kernel!(
6312 #[inline(always)]
6313 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, b: i64x2<Sse4_2>) -> i64x2<Sse4_2> {
6314 _mm_sub_epi64(a.into(), b.into()).simd_into(token)
6315 }
6316 );
6317 kernel(self, a, b)
6318 }
6319 #[inline(always)]
6320 fn saturating_sub_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6321 crate::kernel!(
6322 #[inline(always)]
6323 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, b: i64x2<Sse4_2>) -> i64x2<Sse4_2> {
6324 let a = a.into();
6325 let b = b.into();
6326 let wrapped = _mm_sub_epi64(a, b);
6327 let overflow = _mm_xor_si128(_mm_cmpgt_epi64(wrapped, a), b);
6328 let a_sign = _mm_srai_epi32::<31>(_mm_shuffle_epi32::<0xf5>(a));
6329 let bound = _mm_xor_si128(a_sign, _mm_set1_epi64x(i64::MAX));
6330 let result = _mm_blendv_pd(
6331 _mm_castsi128_pd(wrapped),
6332 _mm_castsi128_pd(bound),
6333 _mm_castsi128_pd(overflow),
6334 );
6335 _mm_castpd_si128(result).simd_into(token)
6336 }
6337 );
6338 kernel(self, a, b)
6339 }
6340 #[inline(always)]
6341 fn mul_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6342 crate::kernel!(
6343 #[inline(always)]
6344 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, b: i64x2<Sse4_2>) -> i64x2<Sse4_2> {
6345 let a = a.into();
6346 let b = b.into();
6347 let a_high = _mm_shuffle_epi32::<0xf5>(a);
6348 let b_high = _mm_shuffle_epi32::<0xf5>(b);
6349 let cross = _mm_add_epi64(_mm_mul_epu32(a_high, b), _mm_mul_epu32(a, b_high));
6350 let low = _mm_mul_epu32(a, b);
6351 _mm_add_epi64(low, _mm_slli_epi64::<32>(cross)).simd_into(token)
6352 }
6353 );
6354 kernel(self, a, b)
6355 }
6356 #[inline(always)]
6357 fn and_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6358 crate::kernel!(
6359 #[inline(always)]
6360 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, b: i64x2<Sse4_2>) -> i64x2<Sse4_2> {
6361 _mm_and_si128(a.into(), b.into()).simd_into(token)
6362 }
6363 );
6364 kernel(self, a, b)
6365 }
6366 #[inline(always)]
6367 fn or_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6368 crate::kernel!(
6369 #[inline(always)]
6370 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, b: i64x2<Sse4_2>) -> i64x2<Sse4_2> {
6371 _mm_or_si128(a.into(), b.into()).simd_into(token)
6372 }
6373 );
6374 kernel(self, a, b)
6375 }
6376 #[inline(always)]
6377 fn xor_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6378 crate::kernel!(
6379 #[inline(always)]
6380 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, b: i64x2<Sse4_2>) -> i64x2<Sse4_2> {
6381 _mm_xor_si128(a.into(), b.into()).simd_into(token)
6382 }
6383 );
6384 kernel(self, a, b)
6385 }
6386 #[inline(always)]
6387 fn not_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6388 a ^ !0
6389 }
6390 #[inline(always)]
6391 fn shl_i64x2(self, a: i64x2<Self>, shift: u32) -> i64x2<Self> {
6392 crate::kernel!(
6393 #[inline(always)]
6394 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, shift: u32) -> i64x2<Sse4_2> {
6395 _mm_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
6396 }
6397 );
6398 kernel(self, a, shift)
6399 }
6400 #[inline(always)]
6401 fn shlv_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6402 [
6403 i64::wrapping_shl(a[0usize], b[0usize] as u32),
6404 i64::wrapping_shl(a[1usize], b[1usize] as u32),
6405 ]
6406 .simd_into(self)
6407 }
6408 #[inline(always)]
6409 fn shr_i64x2(self, a: i64x2<Self>, shift: u32) -> i64x2<Self> {
6410 crate::kernel!(
6411 #[inline(always)]
6412 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, shift: u32) -> i64x2<Sse4_2> {
6413 let value = a.into();
6414 let count = _mm_cvtsi32_si128(shift.cast_signed());
6415 let shifted_bias = _mm_srl_epi64(_mm_set1_epi64x(i64::MIN), count);
6416 let shifted = _mm_srl_epi64(value, count);
6417 _mm_sub_epi64(_mm_xor_si128(shifted, shifted_bias), shifted_bias).simd_into(token)
6418 }
6419 );
6420 kernel(self, a, shift)
6421 }
6422 #[inline(always)]
6423 fn shrv_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6424 [
6425 i64::wrapping_shr(a[0usize], b[0usize] as u32),
6426 i64::wrapping_shr(a[1usize], b[1usize] as u32),
6427 ]
6428 .simd_into(self)
6429 }
6430 #[inline(always)]
6431 fn reduce_max_i64x2(self, a: i64x2<Self>) -> i64 {
6432 crate::kernel!(
6433 #[inline(always)]
6434 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>) -> i64 {
6435 let lanes: [i64; 2] = a.into();
6436 lanes[0].max(lanes[1])
6437 }
6438 );
6439 kernel(self, a)
6440 }
6441 #[inline(always)]
6442 fn reduce_min_i64x2(self, a: i64x2<Self>) -> i64 {
6443 crate::kernel!(
6444 #[inline(always)]
6445 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>) -> i64 {
6446 let lanes: [i64; 2] = a.into();
6447 lanes[0].min(lanes[1])
6448 }
6449 );
6450 kernel(self, a)
6451 }
6452 #[inline(always)]
6453 fn reduce_sum_i64x2(self, a: i64x2<Self>) -> i64 {
6454 crate::kernel!(
6455 #[inline(always)]
6456 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>) -> i64 {
6457 let sum: __m128i = a.into();
6458 let sum = _mm_add_epi64(sum, _mm_srli_si128::<8>(sum));
6459 let lanes: [i64; 2usize] = crate::transmute::checked_transmute_copy(&sum);
6460 lanes[0]
6461 }
6462 );
6463 kernel(self, a)
6464 }
6465 #[inline(always)]
6466 fn reduce_product_i64x2(self, a: i64x2<Self>) -> i64 {
6467 let mul_level_0: [i64; 1usize] = [a[0usize].wrapping_mul(a[1usize])];
6468 mul_level_0[0]
6469 }
6470 #[inline(always)]
6471 fn max_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6472 crate::kernel!(
6473 #[inline(always)]
6474 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, b: i64x2<Sse4_2>) -> i64x2<Sse4_2> {
6475 let a = a.into();
6476 let b = b.into();
6477 let mask = _mm_cmpgt_epi64(b, a);
6478 let result = _mm_blendv_pd(
6479 _mm_castsi128_pd(a),
6480 _mm_castsi128_pd(b),
6481 _mm_castsi128_pd(mask),
6482 );
6483 _mm_castpd_si128(result).simd_into(token)
6484 }
6485 );
6486 kernel(self, a, b)
6487 }
6488 #[inline(always)]
6489 fn min_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6490 crate::kernel!(
6491 #[inline(always)]
6492 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, b: i64x2<Sse4_2>) -> i64x2<Sse4_2> {
6493 let a = a.into();
6494 let b = b.into();
6495 let mask = _mm_cmpgt_epi64(a, b);
6496 let result = _mm_blendv_pd(
6497 _mm_castsi128_pd(a),
6498 _mm_castsi128_pd(b),
6499 _mm_castsi128_pd(mask),
6500 );
6501 _mm_castpd_si128(result).simd_into(token)
6502 }
6503 );
6504 kernel(self, a, b)
6505 }
6506 #[inline(always)]
6507 fn clamp_i64x2(self, a: i64x2<Self>, b: i64x2<Self>, c: i64x2<Self>) -> i64x2<Self> {
6508 debug_assert!(
6509 b.simd_le(c).all_true(),
6510 "clamp requires min <= max for all elements"
6511 );
6512 a.max(b).min(c)
6513 }
6514 #[inline(always)]
6515 fn simd_eq_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
6516 crate::kernel!(
6517 #[inline(always)]
6518 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, b: i64x2<Sse4_2>) -> mask64x2<Sse4_2> {
6519 _mm_cmpeq_epi64(a.into(), b.into()).simd_into(token)
6520 }
6521 );
6522 kernel(self, a, b)
6523 }
6524 #[inline(always)]
6525 fn simd_lt_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
6526 crate::kernel!(
6527 #[inline(always)]
6528 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, b: i64x2<Sse4_2>) -> mask64x2<Sse4_2> {
6529 _mm_cmpgt_epi64(b.into(), a.into()).simd_into(token)
6530 }
6531 );
6532 kernel(self, a, b)
6533 }
6534 #[inline(always)]
6535 fn simd_le_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
6536 crate::kernel!(
6537 #[inline(always)]
6538 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, b: i64x2<Sse4_2>) -> mask64x2<Sse4_2> {
6539 _mm_xor_si128(_mm_cmpgt_epi64(a.into(), b.into()), _mm_set1_epi64x(-1))
6540 .simd_into(token)
6541 }
6542 );
6543 kernel(self, a, b)
6544 }
6545 #[inline(always)]
6546 fn zip_low_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6547 crate::kernel!(
6548 #[inline(always)]
6549 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, b: i64x2<Sse4_2>) -> i64x2<Sse4_2> {
6550 _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token)
6551 }
6552 );
6553 kernel(self, a, b)
6554 }
6555 #[inline(always)]
6556 fn zip_high_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6557 crate::kernel!(
6558 #[inline(always)]
6559 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, b: i64x2<Sse4_2>) -> i64x2<Sse4_2> {
6560 _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token)
6561 }
6562 );
6563 kernel(self, a, b)
6564 }
6565 #[inline(always)]
6566 fn unzip_low_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6567 crate::kernel!(
6568 #[inline(always)]
6569 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, b: i64x2<Sse4_2>) -> i64x2<Sse4_2> {
6570 _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token)
6571 }
6572 );
6573 kernel(self, a, b)
6574 }
6575 #[inline(always)]
6576 fn unzip_high_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6577 crate::kernel!(
6578 #[inline(always)]
6579 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, b: i64x2<Sse4_2>) -> i64x2<Sse4_2> {
6580 _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token)
6581 }
6582 );
6583 kernel(self, a, b)
6584 }
6585 #[inline(always)]
6586 fn interleave_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> (i64x2<Self>, i64x2<Self>) {
6587 (self.zip_low_i64x2(a, b), self.zip_high_i64x2(a, b))
6588 }
6589 #[inline(always)]
6590 fn deinterleave_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> (i64x2<Self>, i64x2<Self>) {
6591 (self.unzip_low_i64x2(a, b), self.unzip_high_i64x2(a, b))
6592 }
6593 #[inline(always)]
6594 fn select_i64x2(self, a: mask64x2<Self>, b: i64x2<Self>, c: i64x2<Self>) -> i64x2<Self> {
6595 crate::kernel!(
6596 #[inline(always)]
6597 fn kernel(
6598 token: Sse4_2,
6599 a: mask64x2<Sse4_2>,
6600 b: i64x2<Sse4_2>,
6601 c: i64x2<Sse4_2>,
6602 ) -> i64x2<Sse4_2> {
6603 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
6604 }
6605 );
6606 kernel(self, a, b, c)
6607 }
6608 #[inline(always)]
6609 fn combine_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x4<Self> {
6610 i64x4 {
6611 val: crate::support::Aligned256([a.val.0, b.val.0]),
6612 simd: self,
6613 }
6614 }
6615 #[inline(always)]
6616 fn neg_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6617 crate::kernel!(
6618 #[inline(always)]
6619 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>) -> i64x2<Sse4_2> {
6620 _mm_sub_epi64(_mm_setzero_si128(), a.into()).simd_into(token)
6621 }
6622 );
6623 kernel(self, a)
6624 }
6625 #[inline(always)]
6626 fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2<Self>; 4usize] {
6627 crate::kernel!(
6628 #[inline(always)]
6629 fn kernel(token: Sse4_2, src: &[i64; 8usize]) -> [i64x2<Sse4_2>; 4usize] {
6630 let (chunks, []) = src.as_chunks::<2usize>() else {
6631 unreachable!()
6632 };
6633 let v0: __m128i =
6634 crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[0]);
6635 let v1: __m128i =
6636 crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[1]);
6637 let v2: __m128i =
6638 crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[2]);
6639 let v3: __m128i =
6640 crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[3]);
6641 let out0 = _mm_unpacklo_epi64(v0, v2);
6642 let out1 = _mm_unpackhi_epi64(v0, v2);
6643 let out2 = _mm_unpacklo_epi64(v1, v3);
6644 let out3 = _mm_unpackhi_epi64(v1, v3);
6645 [
6646 out0.simd_into(token),
6647 out1.simd_into(token),
6648 out2.simd_into(token),
6649 out3.simd_into(token),
6650 ]
6651 }
6652 );
6653 kernel(self, src)
6654 }
6655 #[inline(always)]
6656 fn store_four_interleaved_i64x2(
6657 self,
6658 vectors: [i64x2<Self>; 4usize],
6659 dest: &mut [i64; 8usize],
6660 ) -> () {
6661 crate::kernel!(
6662 #[inline(always)]
6663 fn kernel(
6664 token: Sse4_2,
6665 vectors: [i64x2<Sse4_2>; 4usize],
6666 dest: &mut [i64; 8usize],
6667 ) -> () {
6668 let _ = token;
6669 let v0: __m128i = vectors[0].into();
6670 let v1: __m128i = vectors[1].into();
6671 let v2: __m128i = vectors[2].into();
6672 let v3: __m128i = vectors[3].into();
6673 let out0 = _mm_unpacklo_epi64(v0, v1);
6674 let out1 = _mm_unpacklo_epi64(v2, v3);
6675 let out2 = _mm_unpackhi_epi64(v0, v1);
6676 let out3 = _mm_unpackhi_epi64(v2, v3);
6677 let (chunks, []) = dest.as_chunks_mut::<2usize>() else {
6678 unreachable!()
6679 };
6680 crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>(
6681 out0,
6682 &mut chunks[0],
6683 );
6684 crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>(
6685 out1,
6686 &mut chunks[1],
6687 );
6688 crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>(
6689 out2,
6690 &mut chunks[2],
6691 );
6692 crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>(
6693 out3,
6694 &mut chunks[3],
6695 );
6696 }
6697 );
6698 kernel(self, vectors, dest);
6699 }
6700 #[inline(always)]
6701 fn narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
6702 crate::kernel!(
6703 #[inline(always)]
6704 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, b: i64x2<Sse4_2>) -> i32x4<Sse4_2> {
6705 _mm_castps_si128(_mm_shuffle_ps::<0x88>(
6706 _mm_castsi128_ps(a.into()),
6707 _mm_castsi128_ps(b.into()),
6708 ))
6709 .simd_into(token)
6710 }
6711 );
6712 kernel(self, a, b)
6713 }
6714 #[inline(always)]
6715 fn saturating_narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
6716 crate::kernel!(
6717 #[inline(always)]
6718 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>, b: i64x2<Sse4_2>) -> i32x4<Sse4_2> {
6719 let a = a.into();
6720 let b = b.into();
6721 let low = _mm_castps_si128(_mm_shuffle_ps::<0x88>(
6722 _mm_castsi128_ps(a),
6723 _mm_castsi128_ps(b),
6724 ));
6725 let high = _mm_castps_si128(_mm_shuffle_ps::<0xdd>(
6726 _mm_castsi128_ps(a),
6727 _mm_castsi128_ps(b),
6728 ));
6729 let low_sign = _mm_srai_epi32::<31>(low);
6730 let fits = _mm_cmpeq_epi32(high, low_sign);
6731 let high_sign = _mm_srai_epi32::<31>(high);
6732 let bound = _mm_xor_si128(high_sign, _mm_set1_epi32(i32::MAX));
6733 _mm_blendv_epi8(bound, low, fits).simd_into(token)
6734 }
6735 );
6736 kernel(self, a, b)
6737 }
6738 #[inline(always)]
6739 fn relaxed_narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
6740 debug_assert!(
6741 a.as_slice()
6742 .iter()
6743 .chain(b.as_slice())
6744 .all(|&value| { value >= i32::MIN as i64 && value <= i32::MAX as i64 }),
6745 "relaxed_narrow inputs must fit in the destination type",
6746 );
6747 self.narrow_i64x2(a, b)
6748 }
6749 #[inline(always)]
6750 fn cvt_f64_i64x2(self, a: i64x2<Self>) -> f64x2<Self> {
6751 crate::kernel!(
6752 #[inline(always)]
6753 fn kernel(token: Sse4_2, a: i64x2<Sse4_2>) -> f64x2<Sse4_2> {
6754 let a = a.into();
6755 let low = _mm_blend_epi16::<0xcc>(a, _mm_set1_epi64x(0x4330_0000_0000_0000));
6756 let high = _mm_srli_epi64::<32>(a);
6757 let high = _mm_xor_si128(high, _mm_set1_epi64x(4985484789646622720i64));
6758 let high = _mm_sub_pd(
6759 _mm_castsi128_pd(high),
6760 _mm_set1_pd(f64::from_bits(4985484789647671296u64)),
6761 );
6762 _mm_add_pd(_mm_castsi128_pd(low), high).simd_into(token)
6763 }
6764 );
6765 kernel(self, a)
6766 }
6767 #[inline(always)]
6768 fn splat_u64x2(self, val: u64) -> u64x2<Self> {
6769 crate::kernel!(
6770 #[inline(always)]
6771 fn kernel(token: Sse4_2, val: u64) -> u64x2<Sse4_2> {
6772 _mm_set1_epi64x(val.cast_signed()).simd_into(token)
6773 }
6774 );
6775 kernel(self, val)
6776 }
6777 #[inline(always)]
6778 fn reverse_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
6779 let indices: u8x16<Self> =
6780 [8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7].simd_into(self);
6781 self.swizzle_dyn_u64x2(a, indices)
6782 }
6783 #[inline(always)]
6784 fn slide_u64x2<const SHIFT: usize>(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
6785 if SHIFT >= 2usize {
6786 return b;
6787 }
6788 let result = dyn_alignr_128(
6789 self,
6790 Bytes::to_bytes(b).val.0,
6791 Bytes::to_bytes(a).val.0,
6792 SHIFT * 8usize,
6793 );
6794 Bytes::from_bytes(u8x16 {
6795 val: crate::support::Aligned128(result),
6796 simd: self,
6797 })
6798 }
6799 #[inline(always)]
6800 fn compress_u64x2(self, values: u64x2<Self>, mask: mask64x2<Self>) -> u64x2<Self> {
6801 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
6802 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0);
6803 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
6804 }
6805 #[inline(always)]
6806 fn compress_merge_u64x2(
6807 self,
6808 values: u64x2<Self>,
6809 mask: mask64x2<Self>,
6810 merge: u64x2<Self>,
6811 ) -> u64x2<Self> {
6812 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
6813 let control = u8x16::simd_from(self, crate::support::compact_128::COMPRESS_64[bits].0);
6814 let compressed = self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control);
6815 let inactive = self.simd_lt_i8x16(i8x16::from_bytes(control), i8x16::splat(self, 0));
6816 Bytes::from_bytes(self.select_u8x16(inactive, Bytes::to_bytes(merge), compressed))
6817 }
6818 #[inline(always)]
6819 fn expand_u64x2(self, values: u64x2<Self>, mask: mask64x2<Self>) -> u64x2<Self> {
6820 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
6821 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0);
6822 Bytes::from_bytes(self.swizzle_dyn_precise_u8x16(Bytes::to_bytes(values), control))
6823 }
6824 #[inline(always)]
6825 fn expand_merge_u64x2(
6826 self,
6827 values: u64x2<Self>,
6828 mask: mask64x2<Self>,
6829 merge: u64x2<Self>,
6830 ) -> u64x2<Self> {
6831 let bits = self.to_bitmask_mask64x2(mask) as usize & 3;
6832 let control = u8x16::simd_from(self, crate::support::compact_128::EXPAND_64[bits].0);
6833 let expanded = Bytes::from_bytes(self.swizzle_dyn_u8x16(Bytes::to_bytes(values), control));
6834 self.select_u64x2(mask, expanded, merge)
6835 }
6836 #[inline(always)]
6837 fn count_ones_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
6838 crate::kernel!(
6839 #[inline(always)]
6840 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>) -> u64x2<Sse4_2> {
6841 let value = a.into();
6842 let nibble_mask = _mm_set1_epi8(0x0f);
6843 let lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4);
6844 let low = _mm_and_si128(value, nibble_mask);
6845 let high = _mm_and_si128(_mm_srli_epi16::<4>(value), nibble_mask);
6846 let byte_counts = _mm_add_epi8(
6847 _mm_shuffle_epi8(lookup, low),
6848 _mm_shuffle_epi8(lookup, high),
6849 );
6850 _mm_sad_epu8(byte_counts, _mm_setzero_si128()).simd_into(token)
6851 }
6852 );
6853 kernel(self, a)
6854 }
6855 #[inline(always)]
6856 fn count_zeros_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
6857 self.count_ones_u64x2(self.not_u64x2(a))
6858 }
6859 #[inline(always)]
6860 fn add_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
6861 crate::kernel!(
6862 #[inline(always)]
6863 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, b: u64x2<Sse4_2>) -> u64x2<Sse4_2> {
6864 _mm_add_epi64(a.into(), b.into()).simd_into(token)
6865 }
6866 );
6867 kernel(self, a, b)
6868 }
6869 #[inline(always)]
6870 fn saturating_add_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
6871 crate::kernel!(
6872 #[inline(always)]
6873 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, b: u64x2<Sse4_2>) -> u64x2<Sse4_2> {
6874 let a = a.into();
6875 let b = b.into();
6876 let wrapped = _mm_add_epi64(a, b);
6877 let sign_bias = _mm_set1_epi64x(i64::MIN);
6878 let overflow = _mm_cmpgt_epi64(
6879 _mm_xor_si128(a, sign_bias),
6880 _mm_xor_si128(wrapped, sign_bias),
6881 );
6882 _mm_or_si128(wrapped, overflow).simd_into(token)
6883 }
6884 );
6885 kernel(self, a, b)
6886 }
6887 #[inline(always)]
6888 fn sub_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
6889 crate::kernel!(
6890 #[inline(always)]
6891 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, b: u64x2<Sse4_2>) -> u64x2<Sse4_2> {
6892 _mm_sub_epi64(a.into(), b.into()).simd_into(token)
6893 }
6894 );
6895 kernel(self, a, b)
6896 }
6897 #[inline(always)]
6898 fn saturating_sub_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
6899 crate::kernel!(
6900 #[inline(always)]
6901 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, b: u64x2<Sse4_2>) -> u64x2<Sse4_2> {
6902 let a = a.into();
6903 let b = b.into();
6904 let wrapped = _mm_sub_epi64(a, b);
6905 let sign_bias = _mm_set1_epi64x(i64::MIN);
6906 let no_borrow =
6907 _mm_cmpgt_epi64(_mm_xor_si128(a, sign_bias), _mm_xor_si128(b, sign_bias));
6908 _mm_and_si128(wrapped, no_borrow).simd_into(token)
6909 }
6910 );
6911 kernel(self, a, b)
6912 }
6913 #[inline(always)]
6914 fn mul_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
6915 crate::kernel!(
6916 #[inline(always)]
6917 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, b: u64x2<Sse4_2>) -> u64x2<Sse4_2> {
6918 let a = a.into();
6919 let b = b.into();
6920 let a_high = _mm_shuffle_epi32::<0xf5>(a);
6921 let b_high = _mm_shuffle_epi32::<0xf5>(b);
6922 let cross = _mm_add_epi64(_mm_mul_epu32(a_high, b), _mm_mul_epu32(a, b_high));
6923 let low = _mm_mul_epu32(a, b);
6924 _mm_add_epi64(low, _mm_slli_epi64::<32>(cross)).simd_into(token)
6925 }
6926 );
6927 kernel(self, a, b)
6928 }
6929 #[inline(always)]
6930 fn and_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
6931 crate::kernel!(
6932 #[inline(always)]
6933 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, b: u64x2<Sse4_2>) -> u64x2<Sse4_2> {
6934 _mm_and_si128(a.into(), b.into()).simd_into(token)
6935 }
6936 );
6937 kernel(self, a, b)
6938 }
6939 #[inline(always)]
6940 fn or_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
6941 crate::kernel!(
6942 #[inline(always)]
6943 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, b: u64x2<Sse4_2>) -> u64x2<Sse4_2> {
6944 _mm_or_si128(a.into(), b.into()).simd_into(token)
6945 }
6946 );
6947 kernel(self, a, b)
6948 }
6949 #[inline(always)]
6950 fn xor_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
6951 crate::kernel!(
6952 #[inline(always)]
6953 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, b: u64x2<Sse4_2>) -> u64x2<Sse4_2> {
6954 _mm_xor_si128(a.into(), b.into()).simd_into(token)
6955 }
6956 );
6957 kernel(self, a, b)
6958 }
6959 #[inline(always)]
6960 fn not_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
6961 a ^ !0
6962 }
6963 #[inline(always)]
6964 fn shl_u64x2(self, a: u64x2<Self>, shift: u32) -> u64x2<Self> {
6965 crate::kernel!(
6966 #[inline(always)]
6967 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, shift: u32) -> u64x2<Sse4_2> {
6968 _mm_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
6969 }
6970 );
6971 kernel(self, a, shift)
6972 }
6973 #[inline(always)]
6974 fn shlv_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
6975 [
6976 u64::wrapping_shl(a[0usize], b[0usize] as u32),
6977 u64::wrapping_shl(a[1usize], b[1usize] as u32),
6978 ]
6979 .simd_into(self)
6980 }
6981 #[inline(always)]
6982 fn shr_u64x2(self, a: u64x2<Self>, shift: u32) -> u64x2<Self> {
6983 crate::kernel!(
6984 #[inline(always)]
6985 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, shift: u32) -> u64x2<Sse4_2> {
6986 _mm_srl_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
6987 }
6988 );
6989 kernel(self, a, shift)
6990 }
6991 #[inline(always)]
6992 fn shrv_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
6993 [
6994 u64::wrapping_shr(a[0usize], b[0usize] as u32),
6995 u64::wrapping_shr(a[1usize], b[1usize] as u32),
6996 ]
6997 .simd_into(self)
6998 }
6999 #[inline(always)]
7000 fn reduce_max_u64x2(self, a: u64x2<Self>) -> u64 {
7001 crate::kernel!(
7002 #[inline(always)]
7003 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>) -> u64 {
7004 let lanes: [u64; 2] = a.into();
7005 lanes[0].max(lanes[1])
7006 }
7007 );
7008 kernel(self, a)
7009 }
7010 #[inline(always)]
7011 fn reduce_min_u64x2(self, a: u64x2<Self>) -> u64 {
7012 crate::kernel!(
7013 #[inline(always)]
7014 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>) -> u64 {
7015 let lanes: [u64; 2] = a.into();
7016 lanes[0].min(lanes[1])
7017 }
7018 );
7019 kernel(self, a)
7020 }
7021 #[inline(always)]
7022 fn reduce_sum_u64x2(self, a: u64x2<Self>) -> u64 {
7023 crate::kernel!(
7024 #[inline(always)]
7025 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>) -> u64 {
7026 let sum: __m128i = a.into();
7027 let sum = _mm_add_epi64(sum, _mm_srli_si128::<8>(sum));
7028 let lanes: [u64; 2usize] = crate::transmute::checked_transmute_copy(&sum);
7029 lanes[0]
7030 }
7031 );
7032 kernel(self, a)
7033 }
7034 #[inline(always)]
7035 fn reduce_product_u64x2(self, a: u64x2<Self>) -> u64 {
7036 let mul_level_0: [u64; 1usize] = [a[0usize].wrapping_mul(a[1usize])];
7037 mul_level_0[0]
7038 }
7039 #[inline(always)]
7040 fn max_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7041 crate::kernel!(
7042 #[inline(always)]
7043 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, b: u64x2<Sse4_2>) -> u64x2<Sse4_2> {
7044 let a = a.into();
7045 let b = b.into();
7046 let mask = {
7047 let sign_bit = _mm_set1_epi64x(0x8000000000000000u64.cast_signed());
7048 let lhs_signed = _mm_xor_si128(b, sign_bit);
7049 let rhs_signed = _mm_xor_si128(a, sign_bit);
7050 _mm_cmpgt_epi64(lhs_signed, rhs_signed)
7051 };
7052 let result = _mm_blendv_pd(
7053 _mm_castsi128_pd(a),
7054 _mm_castsi128_pd(b),
7055 _mm_castsi128_pd(mask),
7056 );
7057 _mm_castpd_si128(result).simd_into(token)
7058 }
7059 );
7060 kernel(self, a, b)
7061 }
7062 #[inline(always)]
7063 fn min_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7064 crate::kernel!(
7065 #[inline(always)]
7066 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, b: u64x2<Sse4_2>) -> u64x2<Sse4_2> {
7067 let a = a.into();
7068 let b = b.into();
7069 let mask = {
7070 let sign_bit = _mm_set1_epi64x(0x8000000000000000u64.cast_signed());
7071 let lhs_signed = _mm_xor_si128(a, sign_bit);
7072 let rhs_signed = _mm_xor_si128(b, sign_bit);
7073 _mm_cmpgt_epi64(lhs_signed, rhs_signed)
7074 };
7075 let result = _mm_blendv_pd(
7076 _mm_castsi128_pd(a),
7077 _mm_castsi128_pd(b),
7078 _mm_castsi128_pd(mask),
7079 );
7080 _mm_castpd_si128(result).simd_into(token)
7081 }
7082 );
7083 kernel(self, a, b)
7084 }
7085 #[inline(always)]
7086 fn clamp_u64x2(self, a: u64x2<Self>, b: u64x2<Self>, c: u64x2<Self>) -> u64x2<Self> {
7087 debug_assert!(
7088 b.simd_le(c).all_true(),
7089 "clamp requires min <= max for all elements"
7090 );
7091 a.max(b).min(c)
7092 }
7093 #[inline(always)]
7094 fn simd_eq_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
7095 crate::kernel!(
7096 #[inline(always)]
7097 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, b: u64x2<Sse4_2>) -> mask64x2<Sse4_2> {
7098 _mm_cmpeq_epi64(a.into(), b.into()).simd_into(token)
7099 }
7100 );
7101 kernel(self, a, b)
7102 }
7103 #[inline(always)]
7104 fn simd_lt_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
7105 crate::kernel!(
7106 #[inline(always)]
7107 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, b: u64x2<Sse4_2>) -> mask64x2<Sse4_2> {
7108 {
7109 let sign_bit = _mm_set1_epi64x(0x8000000000000000u64.cast_signed());
7110 let lhs_signed = _mm_xor_si128(b.into(), sign_bit);
7111 let rhs_signed = _mm_xor_si128(a.into(), sign_bit);
7112 _mm_cmpgt_epi64(lhs_signed, rhs_signed)
7113 }
7114 .simd_into(token)
7115 }
7116 );
7117 kernel(self, a, b)
7118 }
7119 #[inline(always)]
7120 fn simd_le_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
7121 crate::kernel!(
7122 #[inline(always)]
7123 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, b: u64x2<Sse4_2>) -> mask64x2<Sse4_2> {
7124 _mm_xor_si128(
7125 {
7126 let sign_bit = _mm_set1_epi64x(0x8000000000000000u64.cast_signed());
7127 let lhs_signed = _mm_xor_si128(a.into(), sign_bit);
7128 let rhs_signed = _mm_xor_si128(b.into(), sign_bit);
7129 _mm_cmpgt_epi64(lhs_signed, rhs_signed)
7130 },
7131 _mm_set1_epi64x(-1),
7132 )
7133 .simd_into(token)
7134 }
7135 );
7136 kernel(self, a, b)
7137 }
7138 #[inline(always)]
7139 fn zip_low_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7140 crate::kernel!(
7141 #[inline(always)]
7142 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, b: u64x2<Sse4_2>) -> u64x2<Sse4_2> {
7143 _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token)
7144 }
7145 );
7146 kernel(self, a, b)
7147 }
7148 #[inline(always)]
7149 fn zip_high_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7150 crate::kernel!(
7151 #[inline(always)]
7152 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, b: u64x2<Sse4_2>) -> u64x2<Sse4_2> {
7153 _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token)
7154 }
7155 );
7156 kernel(self, a, b)
7157 }
7158 #[inline(always)]
7159 fn unzip_low_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7160 crate::kernel!(
7161 #[inline(always)]
7162 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, b: u64x2<Sse4_2>) -> u64x2<Sse4_2> {
7163 _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token)
7164 }
7165 );
7166 kernel(self, a, b)
7167 }
7168 #[inline(always)]
7169 fn unzip_high_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7170 crate::kernel!(
7171 #[inline(always)]
7172 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, b: u64x2<Sse4_2>) -> u64x2<Sse4_2> {
7173 _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token)
7174 }
7175 );
7176 kernel(self, a, b)
7177 }
7178 #[inline(always)]
7179 fn interleave_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> (u64x2<Self>, u64x2<Self>) {
7180 (self.zip_low_u64x2(a, b), self.zip_high_u64x2(a, b))
7181 }
7182 #[inline(always)]
7183 fn deinterleave_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> (u64x2<Self>, u64x2<Self>) {
7184 (self.unzip_low_u64x2(a, b), self.unzip_high_u64x2(a, b))
7185 }
7186 #[inline(always)]
7187 fn select_u64x2(self, a: mask64x2<Self>, b: u64x2<Self>, c: u64x2<Self>) -> u64x2<Self> {
7188 crate::kernel!(
7189 #[inline(always)]
7190 fn kernel(
7191 token: Sse4_2,
7192 a: mask64x2<Sse4_2>,
7193 b: u64x2<Sse4_2>,
7194 c: u64x2<Sse4_2>,
7195 ) -> u64x2<Sse4_2> {
7196 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
7197 }
7198 );
7199 kernel(self, a, b, c)
7200 }
7201 #[inline(always)]
7202 fn combine_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x4<Self> {
7203 u64x4 {
7204 val: crate::support::Aligned256([a.val.0, b.val.0]),
7205 simd: self,
7206 }
7207 }
7208 #[inline(always)]
7209 fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2<Self>; 4usize] {
7210 crate::kernel!(
7211 #[inline(always)]
7212 fn kernel(token: Sse4_2, src: &[u64; 8usize]) -> [u64x2<Sse4_2>; 4usize] {
7213 let (chunks, []) = src.as_chunks::<2usize>() else {
7214 unreachable!()
7215 };
7216 let v0: __m128i =
7217 crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[0]);
7218 let v1: __m128i =
7219 crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[1]);
7220 let v2: __m128i =
7221 crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[2]);
7222 let v3: __m128i =
7223 crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[3]);
7224 let out0 = _mm_unpacklo_epi64(v0, v2);
7225 let out1 = _mm_unpackhi_epi64(v0, v2);
7226 let out2 = _mm_unpacklo_epi64(v1, v3);
7227 let out3 = _mm_unpackhi_epi64(v1, v3);
7228 [
7229 out0.simd_into(token),
7230 out1.simd_into(token),
7231 out2.simd_into(token),
7232 out3.simd_into(token),
7233 ]
7234 }
7235 );
7236 kernel(self, src)
7237 }
7238 #[inline(always)]
7239 fn store_four_interleaved_u64x2(
7240 self,
7241 vectors: [u64x2<Self>; 4usize],
7242 dest: &mut [u64; 8usize],
7243 ) -> () {
7244 crate::kernel!(
7245 #[inline(always)]
7246 fn kernel(
7247 token: Sse4_2,
7248 vectors: [u64x2<Sse4_2>; 4usize],
7249 dest: &mut [u64; 8usize],
7250 ) -> () {
7251 let _ = token;
7252 let v0: __m128i = vectors[0].into();
7253 let v1: __m128i = vectors[1].into();
7254 let v2: __m128i = vectors[2].into();
7255 let v3: __m128i = vectors[3].into();
7256 let out0 = _mm_unpacklo_epi64(v0, v1);
7257 let out1 = _mm_unpacklo_epi64(v2, v3);
7258 let out2 = _mm_unpackhi_epi64(v0, v1);
7259 let out3 = _mm_unpackhi_epi64(v2, v3);
7260 let (chunks, []) = dest.as_chunks_mut::<2usize>() else {
7261 unreachable!()
7262 };
7263 crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>(
7264 out0,
7265 &mut chunks[0],
7266 );
7267 crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>(
7268 out1,
7269 &mut chunks[1],
7270 );
7271 crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>(
7272 out2,
7273 &mut chunks[2],
7274 );
7275 crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>(
7276 out3,
7277 &mut chunks[3],
7278 );
7279 }
7280 );
7281 kernel(self, vectors, dest);
7282 }
7283 #[inline(always)]
7284 fn narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
7285 crate::kernel!(
7286 #[inline(always)]
7287 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, b: u64x2<Sse4_2>) -> u32x4<Sse4_2> {
7288 _mm_castps_si128(_mm_shuffle_ps::<0x88>(
7289 _mm_castsi128_ps(a.into()),
7290 _mm_castsi128_ps(b.into()),
7291 ))
7292 .simd_into(token)
7293 }
7294 );
7295 kernel(self, a, b)
7296 }
7297 #[inline(always)]
7298 fn saturating_narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
7299 crate::kernel!(
7300 #[inline(always)]
7301 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>, b: u64x2<Sse4_2>) -> u32x4<Sse4_2> {
7302 let a = a.into();
7303 let b = b.into();
7304 let low = _mm_castps_si128(_mm_shuffle_ps::<0x88>(
7305 _mm_castsi128_ps(a),
7306 _mm_castsi128_ps(b),
7307 ));
7308 let high = _mm_castps_si128(_mm_shuffle_ps::<0xdd>(
7309 _mm_castsi128_ps(a),
7310 _mm_castsi128_ps(b),
7311 ));
7312 let zero = _mm_setzero_si128();
7313 let fits = _mm_cmpeq_epi32(high, zero);
7314 let ones = _mm_cmpeq_epi32(zero, zero);
7315 _mm_or_si128(low, _mm_xor_si128(fits, ones)).simd_into(token)
7316 }
7317 );
7318 kernel(self, a, b)
7319 }
7320 #[inline(always)]
7321 fn relaxed_narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
7322 debug_assert!(
7323 a.as_slice()
7324 .iter()
7325 .chain(b.as_slice())
7326 .all(|&value| { value >= u32::MIN as u64 && value <= u32::MAX as u64 }),
7327 "relaxed_narrow inputs must fit in the destination type",
7328 );
7329 self.narrow_u64x2(a, b)
7330 }
7331 #[inline(always)]
7332 fn cvt_f64_u64x2(self, a: u64x2<Self>) -> f64x2<Self> {
7333 crate::kernel!(
7334 #[inline(always)]
7335 fn kernel(token: Sse4_2, a: u64x2<Sse4_2>) -> f64x2<Sse4_2> {
7336 let a = a.into();
7337 let low = _mm_blend_epi16::<0xcc>(a, _mm_set1_epi64x(0x4330_0000_0000_0000));
7338 let high = _mm_srli_epi64::<32>(a);
7339 let high = _mm_xor_si128(high, _mm_set1_epi64x(4985484787499139072i64));
7340 let high = _mm_sub_pd(
7341 _mm_castsi128_pd(high),
7342 _mm_set1_pd(f64::from_bits(4985484787500187648u64)),
7343 );
7344 _mm_add_pd(_mm_castsi128_pd(low), high).simd_into(token)
7345 }
7346 );
7347 kernel(self, a)
7348 }
7349 #[inline(always)]
7350 fn splat_mask64x2(self, val: bool) -> mask64x2<Self> {
7351 crate::kernel!(
7352 #[inline(always)]
7353 fn kernel(token: Sse4_2, val: bool) -> mask64x2<Sse4_2> {
7354 let val: i64 = if val { !0 } else { 0 };
7355 _mm_set1_epi64x(val).simd_into(token)
7356 }
7357 );
7358 kernel(self, val)
7359 }
7360 #[inline(always)]
7361 fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2<Self> {
7362 crate::kernel!(
7363 #[inline(always)]
7364 fn kernel(token: Sse4_2, bits: u64) -> mask64x2<Sse4_2> {
7365 {
7366 let bit_lanes = _mm_set1_epi64x(bits.cast_signed());
7367 let bit_mask = _mm_set_epi64x(2, 1);
7368 _mm_cmpeq_epi64(_mm_and_si128(bit_lanes, bit_mask), bit_mask)
7369 }
7370 .simd_into(token)
7371 }
7372 );
7373 kernel(self, bits)
7374 }
7375 #[inline(always)]
7376 fn to_bitmask_mask64x2(self, a: mask64x2<Self>) -> u64 {
7377 crate::kernel!(
7378 #[inline(always)]
7379 fn kernel(token: Sse4_2, a: mask64x2<Sse4_2>) -> u64 {
7380 _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 as u64
7381 }
7382 );
7383 kernel(self, a)
7384 }
7385 #[inline(always)]
7386 fn from_vector_mask64x2(self, vector: i64x2<Self>) -> mask64x2<Self> {
7387 mask64x2 {
7388 val: vector.val,
7389 simd: self,
7390 }
7391 }
7392 #[inline(always)]
7393 fn to_vector_mask64x2(self, a: mask64x2<Self>) -> i64x2<Self> {
7394 i64x2 {
7395 val: a.val,
7396 simd: self,
7397 }
7398 }
7399 #[inline(always)]
7400 fn set_mask64x2(self, a: &mut mask64x2<Self>, index: usize, value: bool) -> () {
7401 assert!(
7402 index < 2usize,
7403 "mask lane index {index} is out of bounds for {} lanes",
7404 2usize
7405 );
7406 let mut lanes: [i64; 2usize] = (*a).into();
7407 lanes[index] = if value { !0 } else { 0 };
7408 *a = lanes.simd_into(self);
7409 }
7410 #[inline(always)]
7411 fn rotate_elements_left_mask64x2<const OFFSET: usize>(
7412 self,
7413 a: mask64x2<Self>,
7414 ) -> mask64x2<Self> {
7415 let int = i64x2 {
7416 val: crate::transmute::checked_transmute_copy(&a.val),
7417 simd: self,
7418 };
7419 let rotated = int.rotate_elements_left::<OFFSET>();
7420 mask64x2 {
7421 val: crate::transmute::checked_transmute_copy(&rotated.val),
7422 simd: self,
7423 }
7424 }
7425 #[inline(always)]
7426 fn rotate_elements_right_mask64x2<const OFFSET: usize>(
7427 self,
7428 a: mask64x2<Self>,
7429 ) -> mask64x2<Self> {
7430 let int = i64x2 {
7431 val: crate::transmute::checked_transmute_copy(&a.val),
7432 simd: self,
7433 };
7434 let rotated = int.rotate_elements_right::<OFFSET>();
7435 mask64x2 {
7436 val: crate::transmute::checked_transmute_copy(&rotated.val),
7437 simd: self,
7438 }
7439 }
7440 #[inline(always)]
7441 fn and_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7442 crate::kernel!(
7443 #[inline(always)]
7444 fn kernel(token: Sse4_2, a: mask64x2<Sse4_2>, b: mask64x2<Sse4_2>) -> mask64x2<Sse4_2> {
7445 _mm_and_si128(a.into(), b.into()).simd_into(token)
7446 }
7447 );
7448 kernel(self, a, b)
7449 }
7450 #[inline(always)]
7451 fn or_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7452 crate::kernel!(
7453 #[inline(always)]
7454 fn kernel(token: Sse4_2, a: mask64x2<Sse4_2>, b: mask64x2<Sse4_2>) -> mask64x2<Sse4_2> {
7455 _mm_or_si128(a.into(), b.into()).simd_into(token)
7456 }
7457 );
7458 kernel(self, a, b)
7459 }
7460 #[inline(always)]
7461 fn xor_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7462 crate::kernel!(
7463 #[inline(always)]
7464 fn kernel(token: Sse4_2, a: mask64x2<Sse4_2>, b: mask64x2<Sse4_2>) -> mask64x2<Sse4_2> {
7465 _mm_xor_si128(a.into(), b.into()).simd_into(token)
7466 }
7467 );
7468 kernel(self, a, b)
7469 }
7470 #[inline(always)]
7471 fn not_mask64x2(self, a: mask64x2<Self>) -> mask64x2<Self> {
7472 self.xor_mask64x2(a, self.splat_mask64x2(true))
7473 }
7474 #[inline(always)]
7475 fn reverse_mask64x2(self, a: mask64x2<Self>) -> mask64x2<Self> {
7476 let lanes = i64x2 {
7477 val: crate::transmute::checked_transmute_copy(&a.val),
7478 simd: self,
7479 };
7480 let reversed = self.reverse_i64x2(lanes);
7481 mask64x2 {
7482 val: crate::transmute::checked_transmute_copy(&reversed.val),
7483 simd: self,
7484 }
7485 }
7486 #[inline(always)]
7487 fn select_mask64x2(
7488 self,
7489 a: mask64x2<Self>,
7490 b: mask64x2<Self>,
7491 c: mask64x2<Self>,
7492 ) -> mask64x2<Self> {
7493 crate::kernel!(
7494 #[inline(always)]
7495 fn kernel(
7496 token: Sse4_2,
7497 a: mask64x2<Sse4_2>,
7498 b: mask64x2<Sse4_2>,
7499 c: mask64x2<Sse4_2>,
7500 ) -> mask64x2<Sse4_2> {
7501 _mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
7502 }
7503 );
7504 kernel(self, a, b, c)
7505 }
7506 #[inline(always)]
7507 fn simd_eq_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7508 crate::kernel!(
7509 #[inline(always)]
7510 fn kernel(token: Sse4_2, a: mask64x2<Sse4_2>, b: mask64x2<Sse4_2>) -> mask64x2<Sse4_2> {
7511 _mm_cmpeq_epi64(a.into(), b.into()).simd_into(token)
7512 }
7513 );
7514 kernel(self, a, b)
7515 }
7516 #[inline(always)]
7517 fn any_true_mask64x2(self, a: mask64x2<Self>) -> bool {
7518 crate::kernel!(
7519 #[inline(always)]
7520 fn kernel(token: Sse4_2, a: mask64x2<Sse4_2>) -> bool {
7521 _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 != 0
7522 }
7523 );
7524 kernel(self, a)
7525 }
7526 #[inline(always)]
7527 fn all_true_mask64x2(self, a: mask64x2<Self>) -> bool {
7528 crate::kernel!(
7529 #[inline(always)]
7530 fn kernel(token: Sse4_2, a: mask64x2<Sse4_2>) -> bool {
7531 _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 == 0b11
7532 }
7533 );
7534 kernel(self, a)
7535 }
7536 #[inline(always)]
7537 fn any_false_mask64x2(self, a: mask64x2<Self>) -> bool {
7538 crate::kernel!(
7539 #[inline(always)]
7540 fn kernel(token: Sse4_2, a: mask64x2<Sse4_2>) -> bool {
7541 _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 != 0b11
7542 }
7543 );
7544 kernel(self, a)
7545 }
7546 #[inline(always)]
7547 fn all_false_mask64x2(self, a: mask64x2<Self>) -> bool {
7548 crate::kernel!(
7549 #[inline(always)]
7550 fn kernel(token: Sse4_2, a: mask64x2<Sse4_2>) -> bool {
7551 _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 == 0
7552 }
7553 );
7554 kernel(self, a)
7555 }
7556 #[inline(always)]
7557 fn combine_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x4<Self> {
7558 mask64x4 {
7559 val: crate::support::Aligned256([a.val.0, b.val.0]),
7560 simd: self,
7561 }
7562 }
7563 #[inline(always)]
7564 fn narrow_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask32x4<Self> {
7565 crate::kernel!(
7566 #[inline(always)]
7567 fn kernel(token: Sse4_2, a: mask64x2<Sse4_2>, b: mask64x2<Sse4_2>) -> mask32x4<Sse4_2> {
7568 _mm_castps_si128(_mm_shuffle_ps::<0x88>(
7569 _mm_castsi128_ps(a.into()),
7570 _mm_castsi128_ps(b.into()),
7571 ))
7572 .simd_into(token)
7573 }
7574 );
7575 kernel(self, a, b)
7576 }
7577 #[inline(always)]
7578 fn slide_f32x8<const SHIFT: usize>(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
7579 if SHIFT >= 8usize {
7580 return b;
7581 }
7582 let result = cross_block_alignr_128x2(
7583 self,
7584 Bytes::to_bytes(b).val.0,
7585 Bytes::to_bytes(a).val.0,
7586 SHIFT * 4usize,
7587 );
7588 Bytes::from_bytes(u8x32 {
7589 val: crate::support::Aligned256(result),
7590 simd: self,
7591 })
7592 }
7593 #[inline(always)]
7594 fn compress_f32x8(self, values: f32x8<Self>, mask: mask32x8<Self>) -> f32x8<Self> {
7595 let mask = mask8x32 {
7596 val: crate::transmute::checked_transmute_copy(&mask.val),
7597 simd: self,
7598 };
7599 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
7600 }
7601 #[inline(always)]
7602 fn compress_merge_f32x8(
7603 self,
7604 values: f32x8<Self>,
7605 mask: mask32x8<Self>,
7606 merge: f32x8<Self>,
7607 ) -> f32x8<Self> {
7608 let mask = mask8x32 {
7609 val: crate::transmute::checked_transmute_copy(&mask.val),
7610 simd: self,
7611 };
7612 Bytes::from_bytes(self.compress_merge_u8x32(
7613 Bytes::to_bytes(values),
7614 mask,
7615 Bytes::to_bytes(merge),
7616 ))
7617 }
7618 #[inline(always)]
7619 fn expand_f32x8(self, values: f32x8<Self>, mask: mask32x8<Self>) -> f32x8<Self> {
7620 let mask = mask8x32 {
7621 val: crate::transmute::checked_transmute_copy(&mask.val),
7622 simd: self,
7623 };
7624 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
7625 }
7626 #[inline(always)]
7627 fn expand_merge_f32x8(
7628 self,
7629 values: f32x8<Self>,
7630 mask: mask32x8<Self>,
7631 merge: f32x8<Self>,
7632 ) -> f32x8<Self> {
7633 let mask = mask8x32 {
7634 val: crate::transmute::checked_transmute_copy(&mask.val),
7635 simd: self,
7636 };
7637 Bytes::from_bytes(self.expand_merge_u8x32(
7638 Bytes::to_bytes(values),
7639 mask,
7640 Bytes::to_bytes(merge),
7641 ))
7642 }
7643 #[inline(always)]
7644 fn simd_ne_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
7645 let (a0, a1) = self.split_f32x8(a);
7646 let (b0, b1) = self.split_f32x8(b);
7647 self.combine_mask32x4(self.simd_ne_f32x4(a0, b0), self.simd_ne_f32x4(a1, b1))
7648 }
7649 #[inline(always)]
7650 fn is_nan_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7651 a.simd_ne(a)
7652 }
7653 #[inline(always)]
7654 fn is_infinite_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7655 a.abs().simd_eq(f32::INFINITY)
7656 }
7657 #[inline(always)]
7658 fn is_finite_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7659 a.abs().simd_lt(f32::INFINITY)
7660 }
7661 #[inline(always)]
7662 fn is_subnormal_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7663 let i: i32x8<Self> = a.bitcast();
7664 (i & 0x7F80_0000_i32).simd_eq(0) & (i & 0x007F_FFFF_i32).simd_ne(0)
7665 }
7666 #[inline(always)]
7667 fn is_normal_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7668 let i: i32x8<Self> = a.bitcast();
7669 let exp = i & 0x7F80_0000_i32;
7670 exp.simd_ne(0) & exp.simd_ne(0x7F80_0000_i32)
7671 }
7672 #[inline(always)]
7673 fn is_sign_positive_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7674 let i: i32x8<Self> = a.bitcast();
7675 (i & i32::MIN).simd_eq(0)
7676 }
7677 #[inline(always)]
7678 fn is_sign_negative_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7679 !a.is_sign_positive()
7680 }
7681 #[inline(always)]
7682 fn combine_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x16<Self> {
7683 f32x16 {
7684 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
7685 simd: self,
7686 }
7687 }
7688 #[inline(always)]
7689 fn split_f32x8(self, a: f32x8<Self>) -> (f32x4<Self>, f32x4<Self>) {
7690 (
7691 f32x4 {
7692 val: crate::support::Aligned128(a.val.0[0]),
7693 simd: self,
7694 },
7695 f32x4 {
7696 val: crate::support::Aligned128(a.val.0[1]),
7697 simd: self,
7698 },
7699 )
7700 }
7701 #[inline(always)]
7702 fn slide_i8x32<const SHIFT: usize>(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
7703 if SHIFT >= 32usize {
7704 return b;
7705 }
7706 let result = cross_block_alignr_128x2(
7707 self,
7708 Bytes::to_bytes(b).val.0,
7709 Bytes::to_bytes(a).val.0,
7710 SHIFT,
7711 );
7712 Bytes::from_bytes(u8x32 {
7713 val: crate::support::Aligned256(result),
7714 simd: self,
7715 })
7716 }
7717 #[inline(always)]
7718 fn compress_i8x32(self, values: i8x32<Self>, mask: mask8x32<Self>) -> i8x32<Self> {
7719 let mask = mask8x32 {
7720 val: crate::transmute::checked_transmute_copy(&mask.val),
7721 simd: self,
7722 };
7723 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
7724 }
7725 #[inline(always)]
7726 fn compress_merge_i8x32(
7727 self,
7728 values: i8x32<Self>,
7729 mask: mask8x32<Self>,
7730 merge: i8x32<Self>,
7731 ) -> i8x32<Self> {
7732 let mask = mask8x32 {
7733 val: crate::transmute::checked_transmute_copy(&mask.val),
7734 simd: self,
7735 };
7736 Bytes::from_bytes(self.compress_merge_u8x32(
7737 Bytes::to_bytes(values),
7738 mask,
7739 Bytes::to_bytes(merge),
7740 ))
7741 }
7742 #[inline(always)]
7743 fn expand_i8x32(self, values: i8x32<Self>, mask: mask8x32<Self>) -> i8x32<Self> {
7744 let mask = mask8x32 {
7745 val: crate::transmute::checked_transmute_copy(&mask.val),
7746 simd: self,
7747 };
7748 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
7749 }
7750 #[inline(always)]
7751 fn expand_merge_i8x32(
7752 self,
7753 values: i8x32<Self>,
7754 mask: mask8x32<Self>,
7755 merge: i8x32<Self>,
7756 ) -> i8x32<Self> {
7757 let mask = mask8x32 {
7758 val: crate::transmute::checked_transmute_copy(&mask.val),
7759 simd: self,
7760 };
7761 Bytes::from_bytes(self.expand_merge_u8x32(
7762 Bytes::to_bytes(values),
7763 mask,
7764 Bytes::to_bytes(merge),
7765 ))
7766 }
7767 #[inline(always)]
7768 fn combine_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x64<Self> {
7769 i8x64 {
7770 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
7771 simd: self,
7772 }
7773 }
7774 #[inline(always)]
7775 fn split_i8x32(self, a: i8x32<Self>) -> (i8x16<Self>, i8x16<Self>) {
7776 (
7777 i8x16 {
7778 val: crate::support::Aligned128(a.val.0[0]),
7779 simd: self,
7780 },
7781 i8x16 {
7782 val: crate::support::Aligned128(a.val.0[1]),
7783 simd: self,
7784 },
7785 )
7786 }
7787 #[inline(always)]
7788 fn slide_u8x32<const SHIFT: usize>(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
7789 if SHIFT >= 32usize {
7790 return b;
7791 }
7792 let result = cross_block_alignr_128x2(
7793 self,
7794 Bytes::to_bytes(b).val.0,
7795 Bytes::to_bytes(a).val.0,
7796 SHIFT,
7797 );
7798 Bytes::from_bytes(u8x32 {
7799 val: crate::support::Aligned256(result),
7800 simd: self,
7801 })
7802 }
7803 #[inline(always)]
7804 fn swizzle_dyn_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
7805 self.swizzle_dyn_precise_u8x32(a, indices)
7806 }
7807 #[inline(always)]
7808 fn swizzle_dyn_precise_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
7809 crate::kernel!(
7810 #[inline(always)]
7811 fn kernel(token: Sse4_2, a: u8x32<Sse4_2>, indices: u8x32<Sse4_2>) -> u8x32<Sse4_2> {
7812 let bytes = Bytes::to_bytes(a);
7813 let (table_low, table_high) = token.split_u8x32(bytes);
7814 let (indices_low, indices_high) = token.split_u8x32(indices);
7815 let high_table_offset = token.splat_u8x16(16);
7816 let output_low_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_low);
7817 let output_low_from_high = token.swizzle_dyn_precise_u8x16(
7818 table_high,
7819 token.sub_u8x16(indices_low, high_table_offset),
7820 );
7821 let output_low = token.or_u8x16(output_low_from_low, output_low_from_high);
7822 let output_high_from_low = token.swizzle_dyn_precise_u8x16(table_low, indices_high);
7823 let output_high_from_high = token.swizzle_dyn_precise_u8x16(
7824 table_high,
7825 token.sub_u8x16(indices_high, high_table_offset),
7826 );
7827 let output_high = token.or_u8x16(output_high_from_low, output_high_from_high);
7828 let result_bytes = token.combine_u8x16(output_low, output_high);
7829 Bytes::from_bytes(result_bytes)
7830 }
7831 );
7832 kernel(self, a, indices)
7833 }
7834 #[inline(always)]
7835 fn concat_swizzle_dyn_u8x32(
7836 self,
7837 a: u8x32<Self>,
7838 b: u8x32<Self>,
7839 indices: u8x32<Self>,
7840 ) -> u8x32<Self> {
7841 self.concat_swizzle_dyn_precise_u8x32(a, b, indices)
7842 }
7843 #[inline(always)]
7844 fn concat_swizzle_dyn_precise_u8x32(
7845 self,
7846 a: u8x32<Self>,
7847 b: u8x32<Self>,
7848 indices: u8x32<Self>,
7849 ) -> u8x32<Self> {
7850 let first_table = Bytes::to_bytes(a);
7851 let second_table = Bytes::to_bytes(b);
7852 let second_table_offset = self.splat_u8x32(32);
7853 let from_first = self.swizzle_dyn_precise_u8x32(first_table, indices);
7854 let from_second = self
7855 .swizzle_dyn_precise_u8x32(second_table, self.sub_u8x32(indices, second_table_offset));
7856 let result_bytes = self.or_u8x32(from_first, from_second);
7857 Bytes::from_bytes(result_bytes)
7858 }
7859 #[inline(always)]
7860 fn compress_u8x32(self, values: u8x32<Self>, mask: mask8x32<Self>) -> u8x32<Self> {
7861 crate::kernel!(
7862 #[inline(always)]
7863 fn kernel(
7864 token: Sse4_2,
7865 values: u8x32<Sse4_2>,
7866 mask: mask8x32<Sse4_2>,
7867 ) -> u8x32<Sse4_2> {
7868 let mask_bits = token.to_bitmask_mask8x32(mask);
7869 let mut output = [0u8; 32];
7870 let mut output_lane = 0;
7871 let block_bits = (mask_bits & 0xffff) as usize;
7872 let low_mask = (block_bits) & 0xff;
7873 let high_mask = (block_bits) >> 8;
7874 let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask];
7875 let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask];
7876 let high_control = high_control | 0x0808_0808_0808_0808;
7877 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
7878 let low_count = low_mask.count_ones() as usize;
7879 let compacted = _mm_shuffle_epi8(values.val.0[0], control);
7880 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count];
7881 let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
7882 let compressed = _mm_shuffle_epi8(compacted, splice);
7883 crate::transmute::checked_transmute_store(
7884 compressed,
7885 output[output_lane..].first_chunk_mut::<16>().unwrap(),
7886 );
7887 output_lane += block_bits.count_ones() as usize;
7888 let block_bits = (mask_bits >> 16 & 0xffff) as usize;
7889 let low_mask = (block_bits) & 0xff;
7890 let high_mask = (block_bits) >> 8;
7891 let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask];
7892 let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask];
7893 let high_control = high_control | 0x0808_0808_0808_0808;
7894 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
7895 let low_count = low_mask.count_ones() as usize;
7896 let compacted = _mm_shuffle_epi8(values.val.0[1], control);
7897 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count];
7898 let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
7899 let compressed = _mm_shuffle_epi8(compacted, splice);
7900 crate::transmute::checked_transmute_store(
7901 compressed,
7902 output[output_lane..].first_chunk_mut::<16>().unwrap(),
7903 );
7904 u8x32::simd_from(token, output)
7905 }
7906 );
7907 kernel(self, values, mask)
7908 }
7909 #[inline(always)]
7910 fn compress_merge_u8x32(
7911 self,
7912 values: u8x32<Self>,
7913 mask: mask8x32<Self>,
7914 merge: u8x32<Self>,
7915 ) -> u8x32<Self> {
7916 crate::kernel!(
7917 #[inline(always)]
7918 fn kernel(
7919 token: Sse4_2,
7920 values: u8x32<Sse4_2>,
7921 mask: mask8x32<Sse4_2>,
7922 merge: u8x32<Sse4_2>,
7923 ) -> u8x32<Sse4_2> {
7924 let mask_bits = token.to_bitmask_mask8x32(mask);
7925 let mut output = [0u8; 32];
7926 let mut output_lane = 0;
7927 let block_bits = (mask_bits & 0xffff) as usize;
7928 let low_mask = (block_bits) & 0xff;
7929 let high_mask = (block_bits) >> 8;
7930 let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask];
7931 let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask];
7932 let high_control = high_control | 0x0808_0808_0808_0808;
7933 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
7934 let low_count = low_mask.count_ones() as usize;
7935 let compacted = _mm_shuffle_epi8(values.val.0[0], control);
7936 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count];
7937 let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
7938 let compressed = _mm_shuffle_epi8(compacted, splice);
7939 crate::transmute::checked_transmute_store(
7940 compressed,
7941 output[output_lane..].first_chunk_mut::<16>().unwrap(),
7942 );
7943 output_lane += block_bits.count_ones() as usize;
7944 let block_bits = (mask_bits >> 16 & 0xffff) as usize;
7945 let low_mask = (block_bits) & 0xff;
7946 let high_mask = (block_bits) >> 8;
7947 let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask];
7948 let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask];
7949 let high_control = high_control | 0x0808_0808_0808_0808;
7950 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
7951 let low_count = low_mask.count_ones() as usize;
7952 let compacted = _mm_shuffle_epi8(values.val.0[1], control);
7953 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count];
7954 let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
7955 let compressed = _mm_shuffle_epi8(compacted, splice);
7956 crate::transmute::checked_transmute_store(
7957 compressed,
7958 output[output_lane..].first_chunk_mut::<16>().unwrap(),
7959 );
7960 output_lane += block_bits.count_ones() as usize;
7961 let remaining = output_lane.saturating_sub(0).min(16);
7962 let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>(
7963 &crate::support::COMPACT_PREFIX_MASKS[remaining],
7964 );
7965 let compressed = crate::transmute::checked_transmute_copy(
7966 output[0..].first_chunk::<16>().unwrap(),
7967 );
7968 let blended = _mm_blendv_epi8(merge.val.0[0], compressed, prefix);
7969 crate::transmute::checked_transmute_store(
7970 blended,
7971 output[0..].first_chunk_mut::<16>().unwrap(),
7972 );
7973 let remaining = output_lane.saturating_sub(16).min(16);
7974 let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>(
7975 &crate::support::COMPACT_PREFIX_MASKS[remaining],
7976 );
7977 let compressed = crate::transmute::checked_transmute_copy(
7978 output[16..].first_chunk::<16>().unwrap(),
7979 );
7980 let blended = _mm_blendv_epi8(merge.val.0[1], compressed, prefix);
7981 crate::transmute::checked_transmute_store(
7982 blended,
7983 output[16..].first_chunk_mut::<16>().unwrap(),
7984 );
7985 u8x32::simd_from(token, output)
7986 }
7987 );
7988 kernel(self, values, mask, merge)
7989 }
7990 #[inline(always)]
7991 fn expand_u8x32(self, values: u8x32<Self>, mask: mask8x32<Self>) -> u8x32<Self> {
7992 crate::kernel!(
7993 #[inline(always)]
7994 fn kernel(
7995 token: Sse4_2,
7996 values: u8x32<Sse4_2>,
7997 mask: mask8x32<Sse4_2>,
7998 ) -> u8x32<Sse4_2> {
7999 let input = <[u8; 32]>::from(values);
8000 let mut output = [0u8; 32];
8001 let mut input_lane = 0;
8002 let block_bits = _mm_movemask_epi8(mask.val.0[0]) as usize;
8003 let low_mask = (block_bits) & 0xff;
8004 let high_mask = (block_bits) >> 8;
8005 let low_count = low_mask.count_ones() as u64;
8006 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
8007 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
8008 let high_base = low_count * 0x0101_0101_0101_0101;
8009 let high_control = high_control + high_base;
8010 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
8011 let packed = crate::transmute::checked_transmute_copy(
8012 input[input_lane..].first_chunk::<16>().unwrap(),
8013 );
8014 let expanded = _mm_shuffle_epi8(packed, control);
8015 let result = expanded;
8016 crate::transmute::checked_transmute_store(
8017 result,
8018 output[0..].first_chunk_mut::<16>().unwrap(),
8019 );
8020 input_lane += block_bits.count_ones() as usize;
8021 let block_bits = _mm_movemask_epi8(mask.val.0[1]) as usize;
8022 let low_mask = (block_bits) & 0xff;
8023 let high_mask = (block_bits) >> 8;
8024 let low_count = low_mask.count_ones() as u64;
8025 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
8026 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
8027 let high_base = low_count * 0x0101_0101_0101_0101;
8028 let high_control = high_control + high_base;
8029 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
8030 let packed = crate::transmute::checked_transmute_copy(
8031 input[input_lane..].first_chunk::<16>().unwrap(),
8032 );
8033 let expanded = _mm_shuffle_epi8(packed, control);
8034 let result = expanded;
8035 crate::transmute::checked_transmute_store(
8036 result,
8037 output[16..].first_chunk_mut::<16>().unwrap(),
8038 );
8039 u8x32::simd_from(token, output)
8040 }
8041 );
8042 kernel(self, values, mask)
8043 }
8044 #[inline(always)]
8045 fn expand_merge_u8x32(
8046 self,
8047 values: u8x32<Self>,
8048 mask: mask8x32<Self>,
8049 merge: u8x32<Self>,
8050 ) -> u8x32<Self> {
8051 crate::kernel!(
8052 #[inline(always)]
8053 fn kernel(
8054 token: Sse4_2,
8055 values: u8x32<Sse4_2>,
8056 mask: mask8x32<Sse4_2>,
8057 merge: u8x32<Sse4_2>,
8058 ) -> u8x32<Sse4_2> {
8059 let input = <[u8; 32]>::from(values);
8060 let mut output = [0u8; 32];
8061 let mut input_lane = 0;
8062 let block_bits = _mm_movemask_epi8(mask.val.0[0]) as usize;
8063 let low_mask = (block_bits) & 0xff;
8064 let high_mask = (block_bits) >> 8;
8065 let low_count = low_mask.count_ones() as u64;
8066 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
8067 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
8068 let high_base = low_count * 0x0101_0101_0101_0101;
8069 let high_control = high_control + high_base;
8070 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
8071 let packed = crate::transmute::checked_transmute_copy(
8072 input[input_lane..].first_chunk::<16>().unwrap(),
8073 );
8074 let expanded = _mm_shuffle_epi8(packed, control);
8075 let result = _mm_blendv_epi8(merge.val.0[0], expanded, mask.val.0[0]);
8076 crate::transmute::checked_transmute_store(
8077 result,
8078 output[0..].first_chunk_mut::<16>().unwrap(),
8079 );
8080 input_lane += block_bits.count_ones() as usize;
8081 let block_bits = _mm_movemask_epi8(mask.val.0[1]) as usize;
8082 let low_mask = (block_bits) & 0xff;
8083 let high_mask = (block_bits) >> 8;
8084 let low_count = low_mask.count_ones() as u64;
8085 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
8086 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
8087 let high_base = low_count * 0x0101_0101_0101_0101;
8088 let high_control = high_control + high_base;
8089 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
8090 let packed = crate::transmute::checked_transmute_copy(
8091 input[input_lane..].first_chunk::<16>().unwrap(),
8092 );
8093 let expanded = _mm_shuffle_epi8(packed, control);
8094 let result = _mm_blendv_epi8(merge.val.0[1], expanded, mask.val.0[1]);
8095 crate::transmute::checked_transmute_store(
8096 result,
8097 output[16..].first_chunk_mut::<16>().unwrap(),
8098 );
8099 u8x32::simd_from(token, output)
8100 }
8101 );
8102 kernel(self, values, mask, merge)
8103 }
8104 #[inline(always)]
8105 fn combine_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x64<Self> {
8106 u8x64 {
8107 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8108 simd: self,
8109 }
8110 }
8111 #[inline(always)]
8112 fn split_u8x32(self, a: u8x32<Self>) -> (u8x16<Self>, u8x16<Self>) {
8113 (
8114 u8x16 {
8115 val: crate::support::Aligned128(a.val.0[0]),
8116 simd: self,
8117 },
8118 u8x16 {
8119 val: crate::support::Aligned128(a.val.0[1]),
8120 simd: self,
8121 },
8122 )
8123 }
8124 #[inline(always)]
8125 fn from_vector_mask8x32(self, vector: i8x32<Self>) -> mask8x32<Self> {
8126 mask8x32 {
8127 val: vector.val,
8128 simd: self,
8129 }
8130 }
8131 #[inline(always)]
8132 fn to_vector_mask8x32(self, a: mask8x32<Self>) -> i8x32<Self> {
8133 i8x32 {
8134 val: a.val,
8135 simd: self,
8136 }
8137 }
8138 #[inline(always)]
8139 fn set_mask8x32(self, a: &mut mask8x32<Self>, index: usize, value: bool) -> () {
8140 assert!(
8141 index < 32usize,
8142 "mask lane index {index} is out of bounds for {} lanes",
8143 32usize
8144 );
8145 let mut lanes: [i8; 32usize] = (*a).into();
8146 lanes[index] = if value { !0 } else { 0 };
8147 *a = lanes.simd_into(self);
8148 }
8149 #[inline(always)]
8150 fn rotate_elements_left_mask8x32<const OFFSET: usize>(
8151 self,
8152 a: mask8x32<Self>,
8153 ) -> mask8x32<Self> {
8154 let int = i8x32 {
8155 val: crate::transmute::checked_transmute_copy(&a.val),
8156 simd: self,
8157 };
8158 let rotated = int.rotate_elements_left::<OFFSET>();
8159 mask8x32 {
8160 val: crate::transmute::checked_transmute_copy(&rotated.val),
8161 simd: self,
8162 }
8163 }
8164 #[inline(always)]
8165 fn rotate_elements_right_mask8x32<const OFFSET: usize>(
8166 self,
8167 a: mask8x32<Self>,
8168 ) -> mask8x32<Self> {
8169 let int = i8x32 {
8170 val: crate::transmute::checked_transmute_copy(&a.val),
8171 simd: self,
8172 };
8173 let rotated = int.rotate_elements_right::<OFFSET>();
8174 mask8x32 {
8175 val: crate::transmute::checked_transmute_copy(&rotated.val),
8176 simd: self,
8177 }
8178 }
8179 #[inline(always)]
8180 fn combine_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x64<Self> {
8181 mask8x64 {
8182 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8183 simd: self,
8184 }
8185 }
8186 #[inline(always)]
8187 fn split_mask8x32(self, a: mask8x32<Self>) -> (mask8x16<Self>, mask8x16<Self>) {
8188 (
8189 mask8x16 {
8190 val: crate::support::Aligned128(a.val.0[0]),
8191 simd: self,
8192 },
8193 mask8x16 {
8194 val: crate::support::Aligned128(a.val.0[1]),
8195 simd: self,
8196 },
8197 )
8198 }
8199 #[inline(always)]
8200 fn slide_i16x16<const SHIFT: usize>(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
8201 if SHIFT >= 16usize {
8202 return b;
8203 }
8204 let result = cross_block_alignr_128x2(
8205 self,
8206 Bytes::to_bytes(b).val.0,
8207 Bytes::to_bytes(a).val.0,
8208 SHIFT * 2usize,
8209 );
8210 Bytes::from_bytes(u8x32 {
8211 val: crate::support::Aligned256(result),
8212 simd: self,
8213 })
8214 }
8215 #[inline(always)]
8216 fn compress_i16x16(self, values: i16x16<Self>, mask: mask16x16<Self>) -> i16x16<Self> {
8217 let mask = mask8x32 {
8218 val: crate::transmute::checked_transmute_copy(&mask.val),
8219 simd: self,
8220 };
8221 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
8222 }
8223 #[inline(always)]
8224 fn compress_merge_i16x16(
8225 self,
8226 values: i16x16<Self>,
8227 mask: mask16x16<Self>,
8228 merge: i16x16<Self>,
8229 ) -> i16x16<Self> {
8230 let mask = mask8x32 {
8231 val: crate::transmute::checked_transmute_copy(&mask.val),
8232 simd: self,
8233 };
8234 Bytes::from_bytes(self.compress_merge_u8x32(
8235 Bytes::to_bytes(values),
8236 mask,
8237 Bytes::to_bytes(merge),
8238 ))
8239 }
8240 #[inline(always)]
8241 fn expand_i16x16(self, values: i16x16<Self>, mask: mask16x16<Self>) -> i16x16<Self> {
8242 let mask = mask8x32 {
8243 val: crate::transmute::checked_transmute_copy(&mask.val),
8244 simd: self,
8245 };
8246 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
8247 }
8248 #[inline(always)]
8249 fn expand_merge_i16x16(
8250 self,
8251 values: i16x16<Self>,
8252 mask: mask16x16<Self>,
8253 merge: i16x16<Self>,
8254 ) -> i16x16<Self> {
8255 let mask = mask8x32 {
8256 val: crate::transmute::checked_transmute_copy(&mask.val),
8257 simd: self,
8258 };
8259 Bytes::from_bytes(self.expand_merge_u8x32(
8260 Bytes::to_bytes(values),
8261 mask,
8262 Bytes::to_bytes(merge),
8263 ))
8264 }
8265 #[inline(always)]
8266 fn combine_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x32<Self> {
8267 i16x32 {
8268 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8269 simd: self,
8270 }
8271 }
8272 #[inline(always)]
8273 fn split_i16x16(self, a: i16x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
8274 (
8275 i16x8 {
8276 val: crate::support::Aligned128(a.val.0[0]),
8277 simd: self,
8278 },
8279 i16x8 {
8280 val: crate::support::Aligned128(a.val.0[1]),
8281 simd: self,
8282 },
8283 )
8284 }
8285 #[inline(always)]
8286 fn slide_u16x16<const SHIFT: usize>(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
8287 if SHIFT >= 16usize {
8288 return b;
8289 }
8290 let result = cross_block_alignr_128x2(
8291 self,
8292 Bytes::to_bytes(b).val.0,
8293 Bytes::to_bytes(a).val.0,
8294 SHIFT * 2usize,
8295 );
8296 Bytes::from_bytes(u8x32 {
8297 val: crate::support::Aligned256(result),
8298 simd: self,
8299 })
8300 }
8301 #[inline(always)]
8302 fn compress_u16x16(self, values: u16x16<Self>, mask: mask16x16<Self>) -> u16x16<Self> {
8303 let mask = mask8x32 {
8304 val: crate::transmute::checked_transmute_copy(&mask.val),
8305 simd: self,
8306 };
8307 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
8308 }
8309 #[inline(always)]
8310 fn compress_merge_u16x16(
8311 self,
8312 values: u16x16<Self>,
8313 mask: mask16x16<Self>,
8314 merge: u16x16<Self>,
8315 ) -> u16x16<Self> {
8316 let mask = mask8x32 {
8317 val: crate::transmute::checked_transmute_copy(&mask.val),
8318 simd: self,
8319 };
8320 Bytes::from_bytes(self.compress_merge_u8x32(
8321 Bytes::to_bytes(values),
8322 mask,
8323 Bytes::to_bytes(merge),
8324 ))
8325 }
8326 #[inline(always)]
8327 fn expand_u16x16(self, values: u16x16<Self>, mask: mask16x16<Self>) -> u16x16<Self> {
8328 let mask = mask8x32 {
8329 val: crate::transmute::checked_transmute_copy(&mask.val),
8330 simd: self,
8331 };
8332 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
8333 }
8334 #[inline(always)]
8335 fn expand_merge_u16x16(
8336 self,
8337 values: u16x16<Self>,
8338 mask: mask16x16<Self>,
8339 merge: u16x16<Self>,
8340 ) -> u16x16<Self> {
8341 let mask = mask8x32 {
8342 val: crate::transmute::checked_transmute_copy(&mask.val),
8343 simd: self,
8344 };
8345 Bytes::from_bytes(self.expand_merge_u8x32(
8346 Bytes::to_bytes(values),
8347 mask,
8348 Bytes::to_bytes(merge),
8349 ))
8350 }
8351 #[inline(always)]
8352 fn combine_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x32<Self> {
8353 u16x32 {
8354 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8355 simd: self,
8356 }
8357 }
8358 #[inline(always)]
8359 fn split_u16x16(self, a: u16x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
8360 (
8361 u16x8 {
8362 val: crate::support::Aligned128(a.val.0[0]),
8363 simd: self,
8364 },
8365 u16x8 {
8366 val: crate::support::Aligned128(a.val.0[1]),
8367 simd: self,
8368 },
8369 )
8370 }
8371 #[inline(always)]
8372 fn to_bitmask_mask16x16(self, a: mask16x16<Self>) -> u64 {
8373 crate::kernel!(
8374 #[inline(always)]
8375 fn kernel(token: Sse4_2, a: mask16x16<Sse4_2>) -> u64 {
8376 {
8377 let packed = _mm_packs_epi16(a.val.0[0], a.val.0[1]);
8378 _mm_movemask_epi8(packed) as u32 as u64
8379 }
8380 }
8381 );
8382 kernel(self, a)
8383 }
8384 #[inline(always)]
8385 fn from_vector_mask16x16(self, vector: i16x16<Self>) -> mask16x16<Self> {
8386 mask16x16 {
8387 val: vector.val,
8388 simd: self,
8389 }
8390 }
8391 #[inline(always)]
8392 fn to_vector_mask16x16(self, a: mask16x16<Self>) -> i16x16<Self> {
8393 i16x16 {
8394 val: a.val,
8395 simd: self,
8396 }
8397 }
8398 #[inline(always)]
8399 fn set_mask16x16(self, a: &mut mask16x16<Self>, index: usize, value: bool) -> () {
8400 assert!(
8401 index < 16usize,
8402 "mask lane index {index} is out of bounds for {} lanes",
8403 16usize
8404 );
8405 let mut lanes: [i16; 16usize] = (*a).into();
8406 lanes[index] = if value { !0 } else { 0 };
8407 *a = lanes.simd_into(self);
8408 }
8409 #[inline(always)]
8410 fn rotate_elements_left_mask16x16<const OFFSET: usize>(
8411 self,
8412 a: mask16x16<Self>,
8413 ) -> mask16x16<Self> {
8414 let int = i16x16 {
8415 val: crate::transmute::checked_transmute_copy(&a.val),
8416 simd: self,
8417 };
8418 let rotated = int.rotate_elements_left::<OFFSET>();
8419 mask16x16 {
8420 val: crate::transmute::checked_transmute_copy(&rotated.val),
8421 simd: self,
8422 }
8423 }
8424 #[inline(always)]
8425 fn rotate_elements_right_mask16x16<const OFFSET: usize>(
8426 self,
8427 a: mask16x16<Self>,
8428 ) -> mask16x16<Self> {
8429 let int = i16x16 {
8430 val: crate::transmute::checked_transmute_copy(&a.val),
8431 simd: self,
8432 };
8433 let rotated = int.rotate_elements_right::<OFFSET>();
8434 mask16x16 {
8435 val: crate::transmute::checked_transmute_copy(&rotated.val),
8436 simd: self,
8437 }
8438 }
8439 #[inline(always)]
8440 fn combine_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x32<Self> {
8441 mask16x32 {
8442 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8443 simd: self,
8444 }
8445 }
8446 #[inline(always)]
8447 fn split_mask16x16(self, a: mask16x16<Self>) -> (mask16x8<Self>, mask16x8<Self>) {
8448 (
8449 mask16x8 {
8450 val: crate::support::Aligned128(a.val.0[0]),
8451 simd: self,
8452 },
8453 mask16x8 {
8454 val: crate::support::Aligned128(a.val.0[1]),
8455 simd: self,
8456 },
8457 )
8458 }
8459 #[inline(always)]
8460 fn slide_i32x8<const SHIFT: usize>(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
8461 if SHIFT >= 8usize {
8462 return b;
8463 }
8464 let result = cross_block_alignr_128x2(
8465 self,
8466 Bytes::to_bytes(b).val.0,
8467 Bytes::to_bytes(a).val.0,
8468 SHIFT * 4usize,
8469 );
8470 Bytes::from_bytes(u8x32 {
8471 val: crate::support::Aligned256(result),
8472 simd: self,
8473 })
8474 }
8475 #[inline(always)]
8476 fn compress_i32x8(self, values: i32x8<Self>, mask: mask32x8<Self>) -> i32x8<Self> {
8477 let mask = mask8x32 {
8478 val: crate::transmute::checked_transmute_copy(&mask.val),
8479 simd: self,
8480 };
8481 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
8482 }
8483 #[inline(always)]
8484 fn compress_merge_i32x8(
8485 self,
8486 values: i32x8<Self>,
8487 mask: mask32x8<Self>,
8488 merge: i32x8<Self>,
8489 ) -> i32x8<Self> {
8490 let mask = mask8x32 {
8491 val: crate::transmute::checked_transmute_copy(&mask.val),
8492 simd: self,
8493 };
8494 Bytes::from_bytes(self.compress_merge_u8x32(
8495 Bytes::to_bytes(values),
8496 mask,
8497 Bytes::to_bytes(merge),
8498 ))
8499 }
8500 #[inline(always)]
8501 fn expand_i32x8(self, values: i32x8<Self>, mask: mask32x8<Self>) -> i32x8<Self> {
8502 let mask = mask8x32 {
8503 val: crate::transmute::checked_transmute_copy(&mask.val),
8504 simd: self,
8505 };
8506 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
8507 }
8508 #[inline(always)]
8509 fn expand_merge_i32x8(
8510 self,
8511 values: i32x8<Self>,
8512 mask: mask32x8<Self>,
8513 merge: i32x8<Self>,
8514 ) -> i32x8<Self> {
8515 let mask = mask8x32 {
8516 val: crate::transmute::checked_transmute_copy(&mask.val),
8517 simd: self,
8518 };
8519 Bytes::from_bytes(self.expand_merge_u8x32(
8520 Bytes::to_bytes(values),
8521 mask,
8522 Bytes::to_bytes(merge),
8523 ))
8524 }
8525 #[inline(always)]
8526 fn combine_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x16<Self> {
8527 i32x16 {
8528 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8529 simd: self,
8530 }
8531 }
8532 #[inline(always)]
8533 fn split_i32x8(self, a: i32x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
8534 (
8535 i32x4 {
8536 val: crate::support::Aligned128(a.val.0[0]),
8537 simd: self,
8538 },
8539 i32x4 {
8540 val: crate::support::Aligned128(a.val.0[1]),
8541 simd: self,
8542 },
8543 )
8544 }
8545 #[inline(always)]
8546 fn slide_u32x8<const SHIFT: usize>(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
8547 if SHIFT >= 8usize {
8548 return b;
8549 }
8550 let result = cross_block_alignr_128x2(
8551 self,
8552 Bytes::to_bytes(b).val.0,
8553 Bytes::to_bytes(a).val.0,
8554 SHIFT * 4usize,
8555 );
8556 Bytes::from_bytes(u8x32 {
8557 val: crate::support::Aligned256(result),
8558 simd: self,
8559 })
8560 }
8561 #[inline(always)]
8562 fn compress_u32x8(self, values: u32x8<Self>, mask: mask32x8<Self>) -> u32x8<Self> {
8563 let mask = mask8x32 {
8564 val: crate::transmute::checked_transmute_copy(&mask.val),
8565 simd: self,
8566 };
8567 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
8568 }
8569 #[inline(always)]
8570 fn compress_merge_u32x8(
8571 self,
8572 values: u32x8<Self>,
8573 mask: mask32x8<Self>,
8574 merge: u32x8<Self>,
8575 ) -> u32x8<Self> {
8576 let mask = mask8x32 {
8577 val: crate::transmute::checked_transmute_copy(&mask.val),
8578 simd: self,
8579 };
8580 Bytes::from_bytes(self.compress_merge_u8x32(
8581 Bytes::to_bytes(values),
8582 mask,
8583 Bytes::to_bytes(merge),
8584 ))
8585 }
8586 #[inline(always)]
8587 fn expand_u32x8(self, values: u32x8<Self>, mask: mask32x8<Self>) -> u32x8<Self> {
8588 let mask = mask8x32 {
8589 val: crate::transmute::checked_transmute_copy(&mask.val),
8590 simd: self,
8591 };
8592 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
8593 }
8594 #[inline(always)]
8595 fn expand_merge_u32x8(
8596 self,
8597 values: u32x8<Self>,
8598 mask: mask32x8<Self>,
8599 merge: u32x8<Self>,
8600 ) -> u32x8<Self> {
8601 let mask = mask8x32 {
8602 val: crate::transmute::checked_transmute_copy(&mask.val),
8603 simd: self,
8604 };
8605 Bytes::from_bytes(self.expand_merge_u8x32(
8606 Bytes::to_bytes(values),
8607 mask,
8608 Bytes::to_bytes(merge),
8609 ))
8610 }
8611 #[inline(always)]
8612 fn combine_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x16<Self> {
8613 u32x16 {
8614 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8615 simd: self,
8616 }
8617 }
8618 #[inline(always)]
8619 fn split_u32x8(self, a: u32x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
8620 (
8621 u32x4 {
8622 val: crate::support::Aligned128(a.val.0[0]),
8623 simd: self,
8624 },
8625 u32x4 {
8626 val: crate::support::Aligned128(a.val.0[1]),
8627 simd: self,
8628 },
8629 )
8630 }
8631 #[inline(always)]
8632 fn from_vector_mask32x8(self, vector: i32x8<Self>) -> mask32x8<Self> {
8633 mask32x8 {
8634 val: vector.val,
8635 simd: self,
8636 }
8637 }
8638 #[inline(always)]
8639 fn to_vector_mask32x8(self, a: mask32x8<Self>) -> i32x8<Self> {
8640 i32x8 {
8641 val: a.val,
8642 simd: self,
8643 }
8644 }
8645 #[inline(always)]
8646 fn set_mask32x8(self, a: &mut mask32x8<Self>, index: usize, value: bool) -> () {
8647 assert!(
8648 index < 8usize,
8649 "mask lane index {index} is out of bounds for {} lanes",
8650 8usize
8651 );
8652 let mut lanes: [i32; 8usize] = (*a).into();
8653 lanes[index] = if value { !0 } else { 0 };
8654 *a = lanes.simd_into(self);
8655 }
8656 #[inline(always)]
8657 fn rotate_elements_left_mask32x8<const OFFSET: usize>(
8658 self,
8659 a: mask32x8<Self>,
8660 ) -> mask32x8<Self> {
8661 let int = i32x8 {
8662 val: crate::transmute::checked_transmute_copy(&a.val),
8663 simd: self,
8664 };
8665 let rotated = int.rotate_elements_left::<OFFSET>();
8666 mask32x8 {
8667 val: crate::transmute::checked_transmute_copy(&rotated.val),
8668 simd: self,
8669 }
8670 }
8671 #[inline(always)]
8672 fn rotate_elements_right_mask32x8<const OFFSET: usize>(
8673 self,
8674 a: mask32x8<Self>,
8675 ) -> mask32x8<Self> {
8676 let int = i32x8 {
8677 val: crate::transmute::checked_transmute_copy(&a.val),
8678 simd: self,
8679 };
8680 let rotated = int.rotate_elements_right::<OFFSET>();
8681 mask32x8 {
8682 val: crate::transmute::checked_transmute_copy(&rotated.val),
8683 simd: self,
8684 }
8685 }
8686 #[inline(always)]
8687 fn combine_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x16<Self> {
8688 mask32x16 {
8689 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8690 simd: self,
8691 }
8692 }
8693 #[inline(always)]
8694 fn split_mask32x8(self, a: mask32x8<Self>) -> (mask32x4<Self>, mask32x4<Self>) {
8695 (
8696 mask32x4 {
8697 val: crate::support::Aligned128(a.val.0[0]),
8698 simd: self,
8699 },
8700 mask32x4 {
8701 val: crate::support::Aligned128(a.val.0[1]),
8702 simd: self,
8703 },
8704 )
8705 }
8706 #[inline(always)]
8707 fn slide_f64x4<const SHIFT: usize>(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
8708 if SHIFT >= 4usize {
8709 return b;
8710 }
8711 let result = cross_block_alignr_128x2(
8712 self,
8713 Bytes::to_bytes(b).val.0,
8714 Bytes::to_bytes(a).val.0,
8715 SHIFT * 8usize,
8716 );
8717 Bytes::from_bytes(u8x32 {
8718 val: crate::support::Aligned256(result),
8719 simd: self,
8720 })
8721 }
8722 #[inline(always)]
8723 fn compress_f64x4(self, values: f64x4<Self>, mask: mask64x4<Self>) -> f64x4<Self> {
8724 let mask = mask8x32 {
8725 val: crate::transmute::checked_transmute_copy(&mask.val),
8726 simd: self,
8727 };
8728 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
8729 }
8730 #[inline(always)]
8731 fn compress_merge_f64x4(
8732 self,
8733 values: f64x4<Self>,
8734 mask: mask64x4<Self>,
8735 merge: f64x4<Self>,
8736 ) -> f64x4<Self> {
8737 let mask = mask8x32 {
8738 val: crate::transmute::checked_transmute_copy(&mask.val),
8739 simd: self,
8740 };
8741 Bytes::from_bytes(self.compress_merge_u8x32(
8742 Bytes::to_bytes(values),
8743 mask,
8744 Bytes::to_bytes(merge),
8745 ))
8746 }
8747 #[inline(always)]
8748 fn expand_f64x4(self, values: f64x4<Self>, mask: mask64x4<Self>) -> f64x4<Self> {
8749 let mask = mask8x32 {
8750 val: crate::transmute::checked_transmute_copy(&mask.val),
8751 simd: self,
8752 };
8753 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
8754 }
8755 #[inline(always)]
8756 fn expand_merge_f64x4(
8757 self,
8758 values: f64x4<Self>,
8759 mask: mask64x4<Self>,
8760 merge: f64x4<Self>,
8761 ) -> f64x4<Self> {
8762 let mask = mask8x32 {
8763 val: crate::transmute::checked_transmute_copy(&mask.val),
8764 simd: self,
8765 };
8766 Bytes::from_bytes(self.expand_merge_u8x32(
8767 Bytes::to_bytes(values),
8768 mask,
8769 Bytes::to_bytes(merge),
8770 ))
8771 }
8772 #[inline(always)]
8773 fn simd_ne_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
8774 let (a0, a1) = self.split_f64x4(a);
8775 let (b0, b1) = self.split_f64x4(b);
8776 self.combine_mask64x2(self.simd_ne_f64x2(a0, b0), self.simd_ne_f64x2(a1, b1))
8777 }
8778 #[inline(always)]
8779 fn is_nan_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8780 a.simd_ne(a)
8781 }
8782 #[inline(always)]
8783 fn is_infinite_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8784 a.abs().simd_eq(f64::INFINITY)
8785 }
8786 #[inline(always)]
8787 fn is_finite_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8788 a.abs().simd_lt(f64::INFINITY)
8789 }
8790 #[inline(always)]
8791 fn is_subnormal_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8792 let i: i64x4<Self> = a.bitcast();
8793 (i & 0x7FF0_0000_0000_0000_i64).simd_eq(0) & (i & 0x000F_FFFF_FFFF_FFFF_i64).simd_ne(0)
8794 }
8795 #[inline(always)]
8796 fn is_normal_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8797 let i: i64x4<Self> = a.bitcast();
8798 let exp = i & 0x7FF0_0000_0000_0000_i64;
8799 exp.simd_ne(0) & exp.simd_ne(0x7FF0_0000_0000_0000_i64)
8800 }
8801 #[inline(always)]
8802 fn is_sign_positive_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8803 let i: i64x4<Self> = a.bitcast();
8804 (i & i64::MIN).simd_eq(0)
8805 }
8806 #[inline(always)]
8807 fn is_sign_negative_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8808 !a.is_sign_positive()
8809 }
8810 #[inline(always)]
8811 fn combine_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x8<Self> {
8812 f64x8 {
8813 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8814 simd: self,
8815 }
8816 }
8817 #[inline(always)]
8818 fn split_f64x4(self, a: f64x4<Self>) -> (f64x2<Self>, f64x2<Self>) {
8819 (
8820 f64x2 {
8821 val: crate::support::Aligned128(a.val.0[0]),
8822 simd: self,
8823 },
8824 f64x2 {
8825 val: crate::support::Aligned128(a.val.0[1]),
8826 simd: self,
8827 },
8828 )
8829 }
8830 #[inline(always)]
8831 fn slide_i64x4<const SHIFT: usize>(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
8832 if SHIFT >= 4usize {
8833 return b;
8834 }
8835 let result = cross_block_alignr_128x2(
8836 self,
8837 Bytes::to_bytes(b).val.0,
8838 Bytes::to_bytes(a).val.0,
8839 SHIFT * 8usize,
8840 );
8841 Bytes::from_bytes(u8x32 {
8842 val: crate::support::Aligned256(result),
8843 simd: self,
8844 })
8845 }
8846 #[inline(always)]
8847 fn compress_i64x4(self, values: i64x4<Self>, mask: mask64x4<Self>) -> i64x4<Self> {
8848 let mask = mask8x32 {
8849 val: crate::transmute::checked_transmute_copy(&mask.val),
8850 simd: self,
8851 };
8852 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
8853 }
8854 #[inline(always)]
8855 fn compress_merge_i64x4(
8856 self,
8857 values: i64x4<Self>,
8858 mask: mask64x4<Self>,
8859 merge: i64x4<Self>,
8860 ) -> i64x4<Self> {
8861 let mask = mask8x32 {
8862 val: crate::transmute::checked_transmute_copy(&mask.val),
8863 simd: self,
8864 };
8865 Bytes::from_bytes(self.compress_merge_u8x32(
8866 Bytes::to_bytes(values),
8867 mask,
8868 Bytes::to_bytes(merge),
8869 ))
8870 }
8871 #[inline(always)]
8872 fn expand_i64x4(self, values: i64x4<Self>, mask: mask64x4<Self>) -> i64x4<Self> {
8873 let mask = mask8x32 {
8874 val: crate::transmute::checked_transmute_copy(&mask.val),
8875 simd: self,
8876 };
8877 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
8878 }
8879 #[inline(always)]
8880 fn expand_merge_i64x4(
8881 self,
8882 values: i64x4<Self>,
8883 mask: mask64x4<Self>,
8884 merge: i64x4<Self>,
8885 ) -> i64x4<Self> {
8886 let mask = mask8x32 {
8887 val: crate::transmute::checked_transmute_copy(&mask.val),
8888 simd: self,
8889 };
8890 Bytes::from_bytes(self.expand_merge_u8x32(
8891 Bytes::to_bytes(values),
8892 mask,
8893 Bytes::to_bytes(merge),
8894 ))
8895 }
8896 #[inline(always)]
8897 fn combine_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x8<Self> {
8898 i64x8 {
8899 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8900 simd: self,
8901 }
8902 }
8903 #[inline(always)]
8904 fn split_i64x4(self, a: i64x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
8905 (
8906 i64x2 {
8907 val: crate::support::Aligned128(a.val.0[0]),
8908 simd: self,
8909 },
8910 i64x2 {
8911 val: crate::support::Aligned128(a.val.0[1]),
8912 simd: self,
8913 },
8914 )
8915 }
8916 #[inline(always)]
8917 fn slide_u64x4<const SHIFT: usize>(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
8918 if SHIFT >= 4usize {
8919 return b;
8920 }
8921 let result = cross_block_alignr_128x2(
8922 self,
8923 Bytes::to_bytes(b).val.0,
8924 Bytes::to_bytes(a).val.0,
8925 SHIFT * 8usize,
8926 );
8927 Bytes::from_bytes(u8x32 {
8928 val: crate::support::Aligned256(result),
8929 simd: self,
8930 })
8931 }
8932 #[inline(always)]
8933 fn compress_u64x4(self, values: u64x4<Self>, mask: mask64x4<Self>) -> u64x4<Self> {
8934 let mask = mask8x32 {
8935 val: crate::transmute::checked_transmute_copy(&mask.val),
8936 simd: self,
8937 };
8938 Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
8939 }
8940 #[inline(always)]
8941 fn compress_merge_u64x4(
8942 self,
8943 values: u64x4<Self>,
8944 mask: mask64x4<Self>,
8945 merge: u64x4<Self>,
8946 ) -> u64x4<Self> {
8947 let mask = mask8x32 {
8948 val: crate::transmute::checked_transmute_copy(&mask.val),
8949 simd: self,
8950 };
8951 Bytes::from_bytes(self.compress_merge_u8x32(
8952 Bytes::to_bytes(values),
8953 mask,
8954 Bytes::to_bytes(merge),
8955 ))
8956 }
8957 #[inline(always)]
8958 fn expand_u64x4(self, values: u64x4<Self>, mask: mask64x4<Self>) -> u64x4<Self> {
8959 let mask = mask8x32 {
8960 val: crate::transmute::checked_transmute_copy(&mask.val),
8961 simd: self,
8962 };
8963 Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
8964 }
8965 #[inline(always)]
8966 fn expand_merge_u64x4(
8967 self,
8968 values: u64x4<Self>,
8969 mask: mask64x4<Self>,
8970 merge: u64x4<Self>,
8971 ) -> u64x4<Self> {
8972 let mask = mask8x32 {
8973 val: crate::transmute::checked_transmute_copy(&mask.val),
8974 simd: self,
8975 };
8976 Bytes::from_bytes(self.expand_merge_u8x32(
8977 Bytes::to_bytes(values),
8978 mask,
8979 Bytes::to_bytes(merge),
8980 ))
8981 }
8982 #[inline(always)]
8983 fn combine_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x8<Self> {
8984 u64x8 {
8985 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8986 simd: self,
8987 }
8988 }
8989 #[inline(always)]
8990 fn split_u64x4(self, a: u64x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
8991 (
8992 u64x2 {
8993 val: crate::support::Aligned128(a.val.0[0]),
8994 simd: self,
8995 },
8996 u64x2 {
8997 val: crate::support::Aligned128(a.val.0[1]),
8998 simd: self,
8999 },
9000 )
9001 }
9002 #[inline(always)]
9003 fn from_vector_mask64x4(self, vector: i64x4<Self>) -> mask64x4<Self> {
9004 mask64x4 {
9005 val: vector.val,
9006 simd: self,
9007 }
9008 }
9009 #[inline(always)]
9010 fn to_vector_mask64x4(self, a: mask64x4<Self>) -> i64x4<Self> {
9011 i64x4 {
9012 val: a.val,
9013 simd: self,
9014 }
9015 }
9016 #[inline(always)]
9017 fn set_mask64x4(self, a: &mut mask64x4<Self>, index: usize, value: bool) -> () {
9018 assert!(
9019 index < 4usize,
9020 "mask lane index {index} is out of bounds for {} lanes",
9021 4usize
9022 );
9023 let mut lanes: [i64; 4usize] = (*a).into();
9024 lanes[index] = if value { !0 } else { 0 };
9025 *a = lanes.simd_into(self);
9026 }
9027 #[inline(always)]
9028 fn rotate_elements_left_mask64x4<const OFFSET: usize>(
9029 self,
9030 a: mask64x4<Self>,
9031 ) -> mask64x4<Self> {
9032 let int = i64x4 {
9033 val: crate::transmute::checked_transmute_copy(&a.val),
9034 simd: self,
9035 };
9036 let rotated = int.rotate_elements_left::<OFFSET>();
9037 mask64x4 {
9038 val: crate::transmute::checked_transmute_copy(&rotated.val),
9039 simd: self,
9040 }
9041 }
9042 #[inline(always)]
9043 fn rotate_elements_right_mask64x4<const OFFSET: usize>(
9044 self,
9045 a: mask64x4<Self>,
9046 ) -> mask64x4<Self> {
9047 let int = i64x4 {
9048 val: crate::transmute::checked_transmute_copy(&a.val),
9049 simd: self,
9050 };
9051 let rotated = int.rotate_elements_right::<OFFSET>();
9052 mask64x4 {
9053 val: crate::transmute::checked_transmute_copy(&rotated.val),
9054 simd: self,
9055 }
9056 }
9057 #[inline(always)]
9058 fn combine_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x8<Self> {
9059 mask64x8 {
9060 val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
9061 simd: self,
9062 }
9063 }
9064 #[inline(always)]
9065 fn split_mask64x4(self, a: mask64x4<Self>) -> (mask64x2<Self>, mask64x2<Self>) {
9066 (
9067 mask64x2 {
9068 val: crate::support::Aligned128(a.val.0[0]),
9069 simd: self,
9070 },
9071 mask64x2 {
9072 val: crate::support::Aligned128(a.val.0[1]),
9073 simd: self,
9074 },
9075 )
9076 }
9077 #[inline(always)]
9078 fn slide_f32x16<const SHIFT: usize>(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
9079 if SHIFT >= 16usize {
9080 return b;
9081 }
9082 let result = cross_block_alignr_128x4(
9083 self,
9084 Bytes::to_bytes(b).val.0,
9085 Bytes::to_bytes(a).val.0,
9086 SHIFT * 4usize,
9087 );
9088 Bytes::from_bytes(u8x64 {
9089 val: crate::support::Aligned512(result),
9090 simd: self,
9091 })
9092 }
9093 #[inline(always)]
9094 fn compress_f32x16(self, values: f32x16<Self>, mask: mask32x16<Self>) -> f32x16<Self> {
9095 let mask = mask8x64 {
9096 val: crate::transmute::checked_transmute_copy(&mask.val),
9097 simd: self,
9098 };
9099 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
9100 }
9101 #[inline(always)]
9102 fn compress_merge_f32x16(
9103 self,
9104 values: f32x16<Self>,
9105 mask: mask32x16<Self>,
9106 merge: f32x16<Self>,
9107 ) -> f32x16<Self> {
9108 let mask = mask8x64 {
9109 val: crate::transmute::checked_transmute_copy(&mask.val),
9110 simd: self,
9111 };
9112 Bytes::from_bytes(self.compress_merge_u8x64(
9113 Bytes::to_bytes(values),
9114 mask,
9115 Bytes::to_bytes(merge),
9116 ))
9117 }
9118 #[inline(always)]
9119 fn expand_f32x16(self, values: f32x16<Self>, mask: mask32x16<Self>) -> f32x16<Self> {
9120 let mask = mask8x64 {
9121 val: crate::transmute::checked_transmute_copy(&mask.val),
9122 simd: self,
9123 };
9124 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
9125 }
9126 #[inline(always)]
9127 fn expand_merge_f32x16(
9128 self,
9129 values: f32x16<Self>,
9130 mask: mask32x16<Self>,
9131 merge: f32x16<Self>,
9132 ) -> f32x16<Self> {
9133 let mask = mask8x64 {
9134 val: crate::transmute::checked_transmute_copy(&mask.val),
9135 simd: self,
9136 };
9137 Bytes::from_bytes(self.expand_merge_u8x64(
9138 Bytes::to_bytes(values),
9139 mask,
9140 Bytes::to_bytes(merge),
9141 ))
9142 }
9143 #[inline(always)]
9144 fn simd_ne_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> mask32x16<Self> {
9145 let (a0, a1) = self.split_f32x16(a);
9146 let (b0, b1) = self.split_f32x16(b);
9147 self.combine_mask32x8(self.simd_ne_f32x8(a0, b0), self.simd_ne_f32x8(a1, b1))
9148 }
9149 #[inline(always)]
9150 fn is_nan_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
9151 a.simd_ne(a)
9152 }
9153 #[inline(always)]
9154 fn is_infinite_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
9155 a.abs().simd_eq(f32::INFINITY)
9156 }
9157 #[inline(always)]
9158 fn is_finite_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
9159 a.abs().simd_lt(f32::INFINITY)
9160 }
9161 #[inline(always)]
9162 fn is_subnormal_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
9163 let i: i32x16<Self> = a.bitcast();
9164 (i & 0x7F80_0000_i32).simd_eq(0) & (i & 0x007F_FFFF_i32).simd_ne(0)
9165 }
9166 #[inline(always)]
9167 fn is_normal_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
9168 let i: i32x16<Self> = a.bitcast();
9169 let exp = i & 0x7F80_0000_i32;
9170 exp.simd_ne(0) & exp.simd_ne(0x7F80_0000_i32)
9171 }
9172 #[inline(always)]
9173 fn is_sign_positive_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
9174 let i: i32x16<Self> = a.bitcast();
9175 (i & i32::MIN).simd_eq(0)
9176 }
9177 #[inline(always)]
9178 fn is_sign_negative_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
9179 !a.is_sign_positive()
9180 }
9181 #[inline(always)]
9182 fn split_f32x16(self, a: f32x16<Self>) -> (f32x8<Self>, f32x8<Self>) {
9183 (
9184 f32x8 {
9185 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
9186 simd: self,
9187 },
9188 f32x8 {
9189 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
9190 simd: self,
9191 },
9192 )
9193 }
9194 #[inline(always)]
9195 fn slide_i8x64<const SHIFT: usize>(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
9196 if SHIFT >= 64usize {
9197 return b;
9198 }
9199 let result = cross_block_alignr_128x4(
9200 self,
9201 Bytes::to_bytes(b).val.0,
9202 Bytes::to_bytes(a).val.0,
9203 SHIFT,
9204 );
9205 Bytes::from_bytes(u8x64 {
9206 val: crate::support::Aligned512(result),
9207 simd: self,
9208 })
9209 }
9210 #[inline(always)]
9211 fn compress_i8x64(self, values: i8x64<Self>, mask: mask8x64<Self>) -> i8x64<Self> {
9212 let mask = mask8x64 {
9213 val: crate::transmute::checked_transmute_copy(&mask.val),
9214 simd: self,
9215 };
9216 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
9217 }
9218 #[inline(always)]
9219 fn compress_merge_i8x64(
9220 self,
9221 values: i8x64<Self>,
9222 mask: mask8x64<Self>,
9223 merge: i8x64<Self>,
9224 ) -> i8x64<Self> {
9225 let mask = mask8x64 {
9226 val: crate::transmute::checked_transmute_copy(&mask.val),
9227 simd: self,
9228 };
9229 Bytes::from_bytes(self.compress_merge_u8x64(
9230 Bytes::to_bytes(values),
9231 mask,
9232 Bytes::to_bytes(merge),
9233 ))
9234 }
9235 #[inline(always)]
9236 fn expand_i8x64(self, values: i8x64<Self>, mask: mask8x64<Self>) -> i8x64<Self> {
9237 let mask = mask8x64 {
9238 val: crate::transmute::checked_transmute_copy(&mask.val),
9239 simd: self,
9240 };
9241 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
9242 }
9243 #[inline(always)]
9244 fn expand_merge_i8x64(
9245 self,
9246 values: i8x64<Self>,
9247 mask: mask8x64<Self>,
9248 merge: i8x64<Self>,
9249 ) -> i8x64<Self> {
9250 let mask = mask8x64 {
9251 val: crate::transmute::checked_transmute_copy(&mask.val),
9252 simd: self,
9253 };
9254 Bytes::from_bytes(self.expand_merge_u8x64(
9255 Bytes::to_bytes(values),
9256 mask,
9257 Bytes::to_bytes(merge),
9258 ))
9259 }
9260 #[inline(always)]
9261 fn split_i8x64(self, a: i8x64<Self>) -> (i8x32<Self>, i8x32<Self>) {
9262 (
9263 i8x32 {
9264 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
9265 simd: self,
9266 },
9267 i8x32 {
9268 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
9269 simd: self,
9270 },
9271 )
9272 }
9273 #[inline(always)]
9274 fn slide_u8x64<const SHIFT: usize>(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
9275 if SHIFT >= 64usize {
9276 return b;
9277 }
9278 let result = cross_block_alignr_128x4(
9279 self,
9280 Bytes::to_bytes(b).val.0,
9281 Bytes::to_bytes(a).val.0,
9282 SHIFT,
9283 );
9284 Bytes::from_bytes(u8x64 {
9285 val: crate::support::Aligned512(result),
9286 simd: self,
9287 })
9288 }
9289 #[inline(always)]
9290 fn swizzle_dyn_u8x64(self, a: u8x64<Self>, indices: u8x64<Self>) -> u8x64<Self> {
9291 self.swizzle_dyn_precise_u8x64(a, indices)
9292 }
9293 #[inline(always)]
9294 fn swizzle_dyn_precise_u8x64(self, a: u8x64<Self>, indices: u8x64<Self>) -> u8x64<Self> {
9295 crate::kernel!(
9296 #[inline(always)]
9297 fn kernel(token: Sse4_2, a: u8x64<Sse4_2>, indices: u8x64<Sse4_2>) -> u8x64<Sse4_2> {
9298 let bytes = Bytes::to_bytes(a);
9299 let (table_low, table_high) = token.split_u8x64(bytes);
9300 let (indices_low, indices_high) = token.split_u8x64(indices);
9301 let high_table_offset = token.splat_u8x32(32);
9302 let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low);
9303 let output_low_from_high = token.swizzle_dyn_precise_u8x32(
9304 table_high,
9305 token.sub_u8x32(indices_low, high_table_offset),
9306 );
9307 let output_low = token.or_u8x32(output_low_from_low, output_low_from_high);
9308 let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high);
9309 let output_high_from_high = token.swizzle_dyn_precise_u8x32(
9310 table_high,
9311 token.sub_u8x32(indices_high, high_table_offset),
9312 );
9313 let output_high = token.or_u8x32(output_high_from_low, output_high_from_high);
9314 let result_bytes = token.combine_u8x32(output_low, output_high);
9315 Bytes::from_bytes(result_bytes)
9316 }
9317 );
9318 kernel(self, a, indices)
9319 }
9320 #[inline(always)]
9321 fn concat_swizzle_dyn_u8x64(
9322 self,
9323 a: u8x64<Self>,
9324 b: u8x64<Self>,
9325 indices: u8x64<Self>,
9326 ) -> u8x64<Self> {
9327 self.concat_swizzle_dyn_precise_u8x64(a, b, indices)
9328 }
9329 #[inline(always)]
9330 fn concat_swizzle_dyn_precise_u8x64(
9331 self,
9332 a: u8x64<Self>,
9333 b: u8x64<Self>,
9334 indices: u8x64<Self>,
9335 ) -> u8x64<Self> {
9336 let first_table = Bytes::to_bytes(a);
9337 let second_table = Bytes::to_bytes(b);
9338 let second_table_offset = self.splat_u8x64(64);
9339 let from_first = self.swizzle_dyn_precise_u8x64(first_table, indices);
9340 let from_second = self
9341 .swizzle_dyn_precise_u8x64(second_table, self.sub_u8x64(indices, second_table_offset));
9342 let result_bytes = self.or_u8x64(from_first, from_second);
9343 Bytes::from_bytes(result_bytes)
9344 }
9345 #[inline(always)]
9346 fn compress_u8x64(self, values: u8x64<Self>, mask: mask8x64<Self>) -> u8x64<Self> {
9347 crate::kernel!(
9348 #[inline(always)]
9349 fn kernel(
9350 token: Sse4_2,
9351 values: u8x64<Sse4_2>,
9352 mask: mask8x64<Sse4_2>,
9353 ) -> u8x64<Sse4_2> {
9354 let mask_bits = token.to_bitmask_mask8x64(mask);
9355 let mut output = [0u8; 64];
9356 let mut output_lane = 0;
9357 let block_bits = (mask_bits & 0xffff) as usize;
9358 let low_mask = (block_bits) & 0xff;
9359 let high_mask = (block_bits) >> 8;
9360 let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask];
9361 let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask];
9362 let high_control = high_control | 0x0808_0808_0808_0808;
9363 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
9364 let low_count = low_mask.count_ones() as usize;
9365 let compacted = _mm_shuffle_epi8(values.val.0[0], control);
9366 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count];
9367 let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
9368 let compressed = _mm_shuffle_epi8(compacted, splice);
9369 crate::transmute::checked_transmute_store(
9370 compressed,
9371 output[output_lane..].first_chunk_mut::<16>().unwrap(),
9372 );
9373 output_lane += block_bits.count_ones() as usize;
9374 let block_bits = (mask_bits >> 16 & 0xffff) as usize;
9375 let low_mask = (block_bits) & 0xff;
9376 let high_mask = (block_bits) >> 8;
9377 let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask];
9378 let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask];
9379 let high_control = high_control | 0x0808_0808_0808_0808;
9380 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
9381 let low_count = low_mask.count_ones() as usize;
9382 let compacted = _mm_shuffle_epi8(values.val.0[1], control);
9383 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count];
9384 let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
9385 let compressed = _mm_shuffle_epi8(compacted, splice);
9386 crate::transmute::checked_transmute_store(
9387 compressed,
9388 output[output_lane..].first_chunk_mut::<16>().unwrap(),
9389 );
9390 output_lane += block_bits.count_ones() as usize;
9391 let block_bits = (mask_bits >> 32 & 0xffff) as usize;
9392 let low_mask = (block_bits) & 0xff;
9393 let high_mask = (block_bits) >> 8;
9394 let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask];
9395 let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask];
9396 let high_control = high_control | 0x0808_0808_0808_0808;
9397 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
9398 let low_count = low_mask.count_ones() as usize;
9399 let compacted = _mm_shuffle_epi8(values.val.0[2], control);
9400 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count];
9401 let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
9402 let compressed = _mm_shuffle_epi8(compacted, splice);
9403 crate::transmute::checked_transmute_store(
9404 compressed,
9405 output[output_lane..].first_chunk_mut::<16>().unwrap(),
9406 );
9407 output_lane += block_bits.count_ones() as usize;
9408 let block_bits = (mask_bits >> 48 & 0xffff) as usize;
9409 let low_mask = (block_bits) & 0xff;
9410 let high_mask = (block_bits) >> 8;
9411 let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask];
9412 let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask];
9413 let high_control = high_control | 0x0808_0808_0808_0808;
9414 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
9415 let low_count = low_mask.count_ones() as usize;
9416 let compacted = _mm_shuffle_epi8(values.val.0[3], control);
9417 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count];
9418 let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
9419 let compressed = _mm_shuffle_epi8(compacted, splice);
9420 crate::transmute::checked_transmute_store(
9421 compressed,
9422 output[output_lane..].first_chunk_mut::<16>().unwrap(),
9423 );
9424 u8x64::simd_from(token, output)
9425 }
9426 );
9427 kernel(self, values, mask)
9428 }
9429 #[inline(always)]
9430 fn compress_merge_u8x64(
9431 self,
9432 values: u8x64<Self>,
9433 mask: mask8x64<Self>,
9434 merge: u8x64<Self>,
9435 ) -> u8x64<Self> {
9436 crate::kernel!(
9437 #[inline(always)]
9438 fn kernel(
9439 token: Sse4_2,
9440 values: u8x64<Sse4_2>,
9441 mask: mask8x64<Sse4_2>,
9442 merge: u8x64<Sse4_2>,
9443 ) -> u8x64<Sse4_2> {
9444 let mask_bits = token.to_bitmask_mask8x64(mask);
9445 let mut output = [0u8; 64];
9446 let mut output_lane = 0;
9447 let block_bits = (mask_bits & 0xffff) as usize;
9448 let low_mask = (block_bits) & 0xff;
9449 let high_mask = (block_bits) >> 8;
9450 let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask];
9451 let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask];
9452 let high_control = high_control | 0x0808_0808_0808_0808;
9453 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
9454 let low_count = low_mask.count_ones() as usize;
9455 let compacted = _mm_shuffle_epi8(values.val.0[0], control);
9456 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count];
9457 let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
9458 let compressed = _mm_shuffle_epi8(compacted, splice);
9459 crate::transmute::checked_transmute_store(
9460 compressed,
9461 output[output_lane..].first_chunk_mut::<16>().unwrap(),
9462 );
9463 output_lane += block_bits.count_ones() as usize;
9464 let block_bits = (mask_bits >> 16 & 0xffff) as usize;
9465 let low_mask = (block_bits) & 0xff;
9466 let high_mask = (block_bits) >> 8;
9467 let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask];
9468 let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask];
9469 let high_control = high_control | 0x0808_0808_0808_0808;
9470 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
9471 let low_count = low_mask.count_ones() as usize;
9472 let compacted = _mm_shuffle_epi8(values.val.0[1], control);
9473 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count];
9474 let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
9475 let compressed = _mm_shuffle_epi8(compacted, splice);
9476 crate::transmute::checked_transmute_store(
9477 compressed,
9478 output[output_lane..].first_chunk_mut::<16>().unwrap(),
9479 );
9480 output_lane += block_bits.count_ones() as usize;
9481 let block_bits = (mask_bits >> 32 & 0xffff) as usize;
9482 let low_mask = (block_bits) & 0xff;
9483 let high_mask = (block_bits) >> 8;
9484 let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask];
9485 let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask];
9486 let high_control = high_control | 0x0808_0808_0808_0808;
9487 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
9488 let low_count = low_mask.count_ones() as usize;
9489 let compacted = _mm_shuffle_epi8(values.val.0[2], control);
9490 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count];
9491 let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
9492 let compressed = _mm_shuffle_epi8(compacted, splice);
9493 crate::transmute::checked_transmute_store(
9494 compressed,
9495 output[output_lane..].first_chunk_mut::<16>().unwrap(),
9496 );
9497 output_lane += block_bits.count_ones() as usize;
9498 let block_bits = (mask_bits >> 48 & 0xffff) as usize;
9499 let low_mask = (block_bits) & 0xff;
9500 let high_mask = (block_bits) >> 8;
9501 let low_control = crate::support::COMPRESS_8_CONTROLS[low_mask];
9502 let high_control = crate::support::COMPRESS_8_CONTROLS[high_mask];
9503 let high_control = high_control | 0x0808_0808_0808_0808;
9504 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
9505 let low_count = low_mask.count_ones() as usize;
9506 let compacted = _mm_shuffle_epi8(values.val.0[3], control);
9507 let splice = crate::support::COMPACT_8_SPLICE_CONTROLS[low_count];
9508 let splice = _mm_set_epi64x((splice >> 64) as i64, splice as i64);
9509 let compressed = _mm_shuffle_epi8(compacted, splice);
9510 crate::transmute::checked_transmute_store(
9511 compressed,
9512 output[output_lane..].first_chunk_mut::<16>().unwrap(),
9513 );
9514 output_lane += block_bits.count_ones() as usize;
9515 let remaining = output_lane.saturating_sub(0).min(16);
9516 let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>(
9517 &crate::support::COMPACT_PREFIX_MASKS[remaining],
9518 );
9519 let compressed = crate::transmute::checked_transmute_copy(
9520 output[0..].first_chunk::<16>().unwrap(),
9521 );
9522 let blended = _mm_blendv_epi8(merge.val.0[0], compressed, prefix);
9523 crate::transmute::checked_transmute_store(
9524 blended,
9525 output[0..].first_chunk_mut::<16>().unwrap(),
9526 );
9527 let remaining = output_lane.saturating_sub(16).min(16);
9528 let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>(
9529 &crate::support::COMPACT_PREFIX_MASKS[remaining],
9530 );
9531 let compressed = crate::transmute::checked_transmute_copy(
9532 output[16..].first_chunk::<16>().unwrap(),
9533 );
9534 let blended = _mm_blendv_epi8(merge.val.0[1], compressed, prefix);
9535 crate::transmute::checked_transmute_store(
9536 blended,
9537 output[16..].first_chunk_mut::<16>().unwrap(),
9538 );
9539 let remaining = output_lane.saturating_sub(32).min(16);
9540 let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>(
9541 &crate::support::COMPACT_PREFIX_MASKS[remaining],
9542 );
9543 let compressed = crate::transmute::checked_transmute_copy(
9544 output[32..].first_chunk::<16>().unwrap(),
9545 );
9546 let blended = _mm_blendv_epi8(merge.val.0[2], compressed, prefix);
9547 crate::transmute::checked_transmute_store(
9548 blended,
9549 output[32..].first_chunk_mut::<16>().unwrap(),
9550 );
9551 let remaining = output_lane.saturating_sub(48).min(16);
9552 let [prefix, _] = crate::transmute::checked_transmute_copy::<_, [__m128i; 2]>(
9553 &crate::support::COMPACT_PREFIX_MASKS[remaining],
9554 );
9555 let compressed = crate::transmute::checked_transmute_copy(
9556 output[48..].first_chunk::<16>().unwrap(),
9557 );
9558 let blended = _mm_blendv_epi8(merge.val.0[3], compressed, prefix);
9559 crate::transmute::checked_transmute_store(
9560 blended,
9561 output[48..].first_chunk_mut::<16>().unwrap(),
9562 );
9563 u8x64::simd_from(token, output)
9564 }
9565 );
9566 kernel(self, values, mask, merge)
9567 }
9568 #[inline(always)]
9569 fn expand_u8x64(self, values: u8x64<Self>, mask: mask8x64<Self>) -> u8x64<Self> {
9570 crate::kernel!(
9571 #[inline(always)]
9572 fn kernel(
9573 token: Sse4_2,
9574 values: u8x64<Sse4_2>,
9575 mask: mask8x64<Sse4_2>,
9576 ) -> u8x64<Sse4_2> {
9577 let input = <[u8; 64]>::from(values);
9578 let mut output = [0u8; 64];
9579 let mut input_lane = 0;
9580 let block_bits = _mm_movemask_epi8(mask.val.0[0]) as usize;
9581 let low_mask = (block_bits) & 0xff;
9582 let high_mask = (block_bits) >> 8;
9583 let low_count = low_mask.count_ones() as u64;
9584 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
9585 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
9586 let high_base = low_count * 0x0101_0101_0101_0101;
9587 let high_control = high_control + high_base;
9588 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
9589 let packed = crate::transmute::checked_transmute_copy(
9590 input[input_lane..].first_chunk::<16>().unwrap(),
9591 );
9592 let expanded = _mm_shuffle_epi8(packed, control);
9593 let result = expanded;
9594 crate::transmute::checked_transmute_store(
9595 result,
9596 output[0..].first_chunk_mut::<16>().unwrap(),
9597 );
9598 input_lane += block_bits.count_ones() as usize;
9599 let block_bits = _mm_movemask_epi8(mask.val.0[1]) as usize;
9600 let low_mask = (block_bits) & 0xff;
9601 let high_mask = (block_bits) >> 8;
9602 let low_count = low_mask.count_ones() as u64;
9603 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
9604 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
9605 let high_base = low_count * 0x0101_0101_0101_0101;
9606 let high_control = high_control + high_base;
9607 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
9608 let packed = crate::transmute::checked_transmute_copy(
9609 input[input_lane..].first_chunk::<16>().unwrap(),
9610 );
9611 let expanded = _mm_shuffle_epi8(packed, control);
9612 let result = expanded;
9613 crate::transmute::checked_transmute_store(
9614 result,
9615 output[16..].first_chunk_mut::<16>().unwrap(),
9616 );
9617 input_lane += block_bits.count_ones() as usize;
9618 let block_bits = _mm_movemask_epi8(mask.val.0[2]) as usize;
9619 let low_mask = (block_bits) & 0xff;
9620 let high_mask = (block_bits) >> 8;
9621 let low_count = low_mask.count_ones() as u64;
9622 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
9623 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
9624 let high_base = low_count * 0x0101_0101_0101_0101;
9625 let high_control = high_control + high_base;
9626 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
9627 let packed = crate::transmute::checked_transmute_copy(
9628 input[input_lane..].first_chunk::<16>().unwrap(),
9629 );
9630 let expanded = _mm_shuffle_epi8(packed, control);
9631 let result = expanded;
9632 crate::transmute::checked_transmute_store(
9633 result,
9634 output[32..].first_chunk_mut::<16>().unwrap(),
9635 );
9636 input_lane += block_bits.count_ones() as usize;
9637 let block_bits = _mm_movemask_epi8(mask.val.0[3]) as usize;
9638 let low_mask = (block_bits) & 0xff;
9639 let high_mask = (block_bits) >> 8;
9640 let low_count = low_mask.count_ones() as u64;
9641 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
9642 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
9643 let high_base = low_count * 0x0101_0101_0101_0101;
9644 let high_control = high_control + high_base;
9645 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
9646 let packed = crate::transmute::checked_transmute_copy(
9647 input[input_lane..].first_chunk::<16>().unwrap(),
9648 );
9649 let expanded = _mm_shuffle_epi8(packed, control);
9650 let result = expanded;
9651 crate::transmute::checked_transmute_store(
9652 result,
9653 output[48..].first_chunk_mut::<16>().unwrap(),
9654 );
9655 u8x64::simd_from(token, output)
9656 }
9657 );
9658 kernel(self, values, mask)
9659 }
9660 #[inline(always)]
9661 fn expand_merge_u8x64(
9662 self,
9663 values: u8x64<Self>,
9664 mask: mask8x64<Self>,
9665 merge: u8x64<Self>,
9666 ) -> u8x64<Self> {
9667 crate::kernel!(
9668 #[inline(always)]
9669 fn kernel(
9670 token: Sse4_2,
9671 values: u8x64<Sse4_2>,
9672 mask: mask8x64<Sse4_2>,
9673 merge: u8x64<Sse4_2>,
9674 ) -> u8x64<Sse4_2> {
9675 let input = <[u8; 64]>::from(values);
9676 let mut output = [0u8; 64];
9677 let mut input_lane = 0;
9678 let block_bits = _mm_movemask_epi8(mask.val.0[0]) as usize;
9679 let low_mask = (block_bits) & 0xff;
9680 let high_mask = (block_bits) >> 8;
9681 let low_count = low_mask.count_ones() as u64;
9682 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
9683 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
9684 let high_base = low_count * 0x0101_0101_0101_0101;
9685 let high_control = high_control + high_base;
9686 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
9687 let packed = crate::transmute::checked_transmute_copy(
9688 input[input_lane..].first_chunk::<16>().unwrap(),
9689 );
9690 let expanded = _mm_shuffle_epi8(packed, control);
9691 let result = _mm_blendv_epi8(merge.val.0[0], expanded, mask.val.0[0]);
9692 crate::transmute::checked_transmute_store(
9693 result,
9694 output[0..].first_chunk_mut::<16>().unwrap(),
9695 );
9696 input_lane += block_bits.count_ones() as usize;
9697 let block_bits = _mm_movemask_epi8(mask.val.0[1]) as usize;
9698 let low_mask = (block_bits) & 0xff;
9699 let high_mask = (block_bits) >> 8;
9700 let low_count = low_mask.count_ones() as u64;
9701 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
9702 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
9703 let high_base = low_count * 0x0101_0101_0101_0101;
9704 let high_control = high_control + high_base;
9705 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
9706 let packed = crate::transmute::checked_transmute_copy(
9707 input[input_lane..].first_chunk::<16>().unwrap(),
9708 );
9709 let expanded = _mm_shuffle_epi8(packed, control);
9710 let result = _mm_blendv_epi8(merge.val.0[1], expanded, mask.val.0[1]);
9711 crate::transmute::checked_transmute_store(
9712 result,
9713 output[16..].first_chunk_mut::<16>().unwrap(),
9714 );
9715 input_lane += block_bits.count_ones() as usize;
9716 let block_bits = _mm_movemask_epi8(mask.val.0[2]) as usize;
9717 let low_mask = (block_bits) & 0xff;
9718 let high_mask = (block_bits) >> 8;
9719 let low_count = low_mask.count_ones() as u64;
9720 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
9721 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
9722 let high_base = low_count * 0x0101_0101_0101_0101;
9723 let high_control = high_control + high_base;
9724 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
9725 let packed = crate::transmute::checked_transmute_copy(
9726 input[input_lane..].first_chunk::<16>().unwrap(),
9727 );
9728 let expanded = _mm_shuffle_epi8(packed, control);
9729 let result = _mm_blendv_epi8(merge.val.0[2], expanded, mask.val.0[2]);
9730 crate::transmute::checked_transmute_store(
9731 result,
9732 output[32..].first_chunk_mut::<16>().unwrap(),
9733 );
9734 input_lane += block_bits.count_ones() as usize;
9735 let block_bits = _mm_movemask_epi8(mask.val.0[3]) as usize;
9736 let low_mask = (block_bits) & 0xff;
9737 let high_mask = (block_bits) >> 8;
9738 let low_count = low_mask.count_ones() as u64;
9739 let low_control = crate::support::EXPAND_8_CONTROLS[low_mask];
9740 let high_control = crate::support::EXPAND_8_CONTROLS[high_mask];
9741 let high_base = low_count * 0x0101_0101_0101_0101;
9742 let high_control = high_control + high_base;
9743 let control = _mm_set_epi64x(high_control.cast_signed(), low_control.cast_signed());
9744 let packed = crate::transmute::checked_transmute_copy(
9745 input[input_lane..].first_chunk::<16>().unwrap(),
9746 );
9747 let expanded = _mm_shuffle_epi8(packed, control);
9748 let result = _mm_blendv_epi8(merge.val.0[3], expanded, mask.val.0[3]);
9749 crate::transmute::checked_transmute_store(
9750 result,
9751 output[48..].first_chunk_mut::<16>().unwrap(),
9752 );
9753 u8x64::simd_from(token, output)
9754 }
9755 );
9756 kernel(self, values, mask, merge)
9757 }
9758 #[inline(always)]
9759 fn split_u8x64(self, a: u8x64<Self>) -> (u8x32<Self>, u8x32<Self>) {
9760 (
9761 u8x32 {
9762 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
9763 simd: self,
9764 },
9765 u8x32 {
9766 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
9767 simd: self,
9768 },
9769 )
9770 }
9771 #[inline(always)]
9772 fn from_bitmask_mask8x64(self, bits: u64) -> mask8x64<Self> {
9773 crate::kernel!(
9774 #[inline(always)]
9775 fn kernel(token: Sse4_2, bits: u64) -> mask8x64<Sse4_2> {
9776 {
9777 let bit_bytes = _mm_set1_epi64x(bits.cast_signed());
9778 let bit_mask =
9779 _mm_setr_epi8(1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128);
9780 mask8x64 {
9781 val: crate::support::Aligned512([
9782 {
9783 let bit_bytes = _mm_shuffle_epi8(
9784 bit_bytes,
9785 _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1),
9786 );
9787 _mm_cmpeq_epi8(_mm_and_si128(bit_bytes, bit_mask), bit_mask)
9788 },
9789 {
9790 let bit_bytes = _mm_shuffle_epi8(
9791 bit_bytes,
9792 _mm_setr_epi8(2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3),
9793 );
9794 _mm_cmpeq_epi8(_mm_and_si128(bit_bytes, bit_mask), bit_mask)
9795 },
9796 {
9797 let bit_bytes = _mm_shuffle_epi8(
9798 bit_bytes,
9799 _mm_setr_epi8(4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5),
9800 );
9801 _mm_cmpeq_epi8(_mm_and_si128(bit_bytes, bit_mask), bit_mask)
9802 },
9803 {
9804 let bit_bytes = _mm_shuffle_epi8(
9805 bit_bytes,
9806 _mm_setr_epi8(6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7),
9807 );
9808 _mm_cmpeq_epi8(_mm_and_si128(bit_bytes, bit_mask), bit_mask)
9809 },
9810 ]),
9811 simd: token,
9812 }
9813 }
9814 }
9815 );
9816 kernel(self, bits)
9817 }
9818 #[inline(always)]
9819 fn from_vector_mask8x64(self, vector: i8x64<Self>) -> mask8x64<Self> {
9820 mask8x64 {
9821 val: vector.val,
9822 simd: self,
9823 }
9824 }
9825 #[inline(always)]
9826 fn to_vector_mask8x64(self, a: mask8x64<Self>) -> i8x64<Self> {
9827 i8x64 {
9828 val: a.val,
9829 simd: self,
9830 }
9831 }
9832 #[inline(always)]
9833 fn set_mask8x64(self, a: &mut mask8x64<Self>, index: usize, value: bool) -> () {
9834 assert!(
9835 index < 64usize,
9836 "mask lane index {index} is out of bounds for {} lanes",
9837 64usize
9838 );
9839 let mut lanes: [i8; 64usize] = (*a).into();
9840 lanes[index] = if value { !0 } else { 0 };
9841 *a = lanes.simd_into(self);
9842 }
9843 #[inline(always)]
9844 fn rotate_elements_left_mask8x64<const OFFSET: usize>(
9845 self,
9846 a: mask8x64<Self>,
9847 ) -> mask8x64<Self> {
9848 let int = i8x64 {
9849 val: crate::transmute::checked_transmute_copy(&a.val),
9850 simd: self,
9851 };
9852 let rotated = int.rotate_elements_left::<OFFSET>();
9853 mask8x64 {
9854 val: crate::transmute::checked_transmute_copy(&rotated.val),
9855 simd: self,
9856 }
9857 }
9858 #[inline(always)]
9859 fn rotate_elements_right_mask8x64<const OFFSET: usize>(
9860 self,
9861 a: mask8x64<Self>,
9862 ) -> mask8x64<Self> {
9863 let int = i8x64 {
9864 val: crate::transmute::checked_transmute_copy(&a.val),
9865 simd: self,
9866 };
9867 let rotated = int.rotate_elements_right::<OFFSET>();
9868 mask8x64 {
9869 val: crate::transmute::checked_transmute_copy(&rotated.val),
9870 simd: self,
9871 }
9872 }
9873 #[inline(always)]
9874 fn split_mask8x64(self, a: mask8x64<Self>) -> (mask8x32<Self>, mask8x32<Self>) {
9875 (
9876 mask8x32 {
9877 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
9878 simd: self,
9879 },
9880 mask8x32 {
9881 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
9882 simd: self,
9883 },
9884 )
9885 }
9886 #[inline(always)]
9887 fn slide_i16x32<const SHIFT: usize>(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
9888 if SHIFT >= 32usize {
9889 return b;
9890 }
9891 let result = cross_block_alignr_128x4(
9892 self,
9893 Bytes::to_bytes(b).val.0,
9894 Bytes::to_bytes(a).val.0,
9895 SHIFT * 2usize,
9896 );
9897 Bytes::from_bytes(u8x64 {
9898 val: crate::support::Aligned512(result),
9899 simd: self,
9900 })
9901 }
9902 #[inline(always)]
9903 fn compress_i16x32(self, values: i16x32<Self>, mask: mask16x32<Self>) -> i16x32<Self> {
9904 let mask = mask8x64 {
9905 val: crate::transmute::checked_transmute_copy(&mask.val),
9906 simd: self,
9907 };
9908 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
9909 }
9910 #[inline(always)]
9911 fn compress_merge_i16x32(
9912 self,
9913 values: i16x32<Self>,
9914 mask: mask16x32<Self>,
9915 merge: i16x32<Self>,
9916 ) -> i16x32<Self> {
9917 let mask = mask8x64 {
9918 val: crate::transmute::checked_transmute_copy(&mask.val),
9919 simd: self,
9920 };
9921 Bytes::from_bytes(self.compress_merge_u8x64(
9922 Bytes::to_bytes(values),
9923 mask,
9924 Bytes::to_bytes(merge),
9925 ))
9926 }
9927 #[inline(always)]
9928 fn expand_i16x32(self, values: i16x32<Self>, mask: mask16x32<Self>) -> i16x32<Self> {
9929 let mask = mask8x64 {
9930 val: crate::transmute::checked_transmute_copy(&mask.val),
9931 simd: self,
9932 };
9933 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
9934 }
9935 #[inline(always)]
9936 fn expand_merge_i16x32(
9937 self,
9938 values: i16x32<Self>,
9939 mask: mask16x32<Self>,
9940 merge: i16x32<Self>,
9941 ) -> i16x32<Self> {
9942 let mask = mask8x64 {
9943 val: crate::transmute::checked_transmute_copy(&mask.val),
9944 simd: self,
9945 };
9946 Bytes::from_bytes(self.expand_merge_u8x64(
9947 Bytes::to_bytes(values),
9948 mask,
9949 Bytes::to_bytes(merge),
9950 ))
9951 }
9952 #[inline(always)]
9953 fn split_i16x32(self, a: i16x32<Self>) -> (i16x16<Self>, i16x16<Self>) {
9954 (
9955 i16x16 {
9956 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
9957 simd: self,
9958 },
9959 i16x16 {
9960 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
9961 simd: self,
9962 },
9963 )
9964 }
9965 #[inline(always)]
9966 fn slide_u16x32<const SHIFT: usize>(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
9967 if SHIFT >= 32usize {
9968 return b;
9969 }
9970 let result = cross_block_alignr_128x4(
9971 self,
9972 Bytes::to_bytes(b).val.0,
9973 Bytes::to_bytes(a).val.0,
9974 SHIFT * 2usize,
9975 );
9976 Bytes::from_bytes(u8x64 {
9977 val: crate::support::Aligned512(result),
9978 simd: self,
9979 })
9980 }
9981 #[inline(always)]
9982 fn compress_u16x32(self, values: u16x32<Self>, mask: mask16x32<Self>) -> u16x32<Self> {
9983 let mask = mask8x64 {
9984 val: crate::transmute::checked_transmute_copy(&mask.val),
9985 simd: self,
9986 };
9987 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
9988 }
9989 #[inline(always)]
9990 fn compress_merge_u16x32(
9991 self,
9992 values: u16x32<Self>,
9993 mask: mask16x32<Self>,
9994 merge: u16x32<Self>,
9995 ) -> u16x32<Self> {
9996 let mask = mask8x64 {
9997 val: crate::transmute::checked_transmute_copy(&mask.val),
9998 simd: self,
9999 };
10000 Bytes::from_bytes(self.compress_merge_u8x64(
10001 Bytes::to_bytes(values),
10002 mask,
10003 Bytes::to_bytes(merge),
10004 ))
10005 }
10006 #[inline(always)]
10007 fn expand_u16x32(self, values: u16x32<Self>, mask: mask16x32<Self>) -> u16x32<Self> {
10008 let mask = mask8x64 {
10009 val: crate::transmute::checked_transmute_copy(&mask.val),
10010 simd: self,
10011 };
10012 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
10013 }
10014 #[inline(always)]
10015 fn expand_merge_u16x32(
10016 self,
10017 values: u16x32<Self>,
10018 mask: mask16x32<Self>,
10019 merge: u16x32<Self>,
10020 ) -> u16x32<Self> {
10021 let mask = mask8x64 {
10022 val: crate::transmute::checked_transmute_copy(&mask.val),
10023 simd: self,
10024 };
10025 Bytes::from_bytes(self.expand_merge_u8x64(
10026 Bytes::to_bytes(values),
10027 mask,
10028 Bytes::to_bytes(merge),
10029 ))
10030 }
10031 #[inline(always)]
10032 fn split_u16x32(self, a: u16x32<Self>) -> (u16x16<Self>, u16x16<Self>) {
10033 (
10034 u16x16 {
10035 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
10036 simd: self,
10037 },
10038 u16x16 {
10039 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
10040 simd: self,
10041 },
10042 )
10043 }
10044 #[inline(always)]
10045 fn to_bitmask_mask16x32(self, a: mask16x32<Self>) -> u64 {
10046 crate::kernel!(
10047 #[inline(always)]
10048 fn kernel(token: Sse4_2, a: mask16x32<Sse4_2>) -> u64 {
10049 {
10050 let lo = _mm_packs_epi16(a.val.0[0], a.val.0[1]);
10051 let hi = _mm_packs_epi16(a.val.0[2], a.val.0[3]);
10052 let lo = _mm_movemask_epi8(lo) as u32 as u64;
10053 let hi = _mm_movemask_epi8(hi) as u32 as u64;
10054 lo | (hi << 16usize)
10055 }
10056 }
10057 );
10058 kernel(self, a)
10059 }
10060 #[inline(always)]
10061 fn from_vector_mask16x32(self, vector: i16x32<Self>) -> mask16x32<Self> {
10062 mask16x32 {
10063 val: vector.val,
10064 simd: self,
10065 }
10066 }
10067 #[inline(always)]
10068 fn to_vector_mask16x32(self, a: mask16x32<Self>) -> i16x32<Self> {
10069 i16x32 {
10070 val: a.val,
10071 simd: self,
10072 }
10073 }
10074 #[inline(always)]
10075 fn set_mask16x32(self, a: &mut mask16x32<Self>, index: usize, value: bool) -> () {
10076 assert!(
10077 index < 32usize,
10078 "mask lane index {index} is out of bounds for {} lanes",
10079 32usize
10080 );
10081 let mut lanes: [i16; 32usize] = (*a).into();
10082 lanes[index] = if value { !0 } else { 0 };
10083 *a = lanes.simd_into(self);
10084 }
10085 #[inline(always)]
10086 fn rotate_elements_left_mask16x32<const OFFSET: usize>(
10087 self,
10088 a: mask16x32<Self>,
10089 ) -> mask16x32<Self> {
10090 let int = i16x32 {
10091 val: crate::transmute::checked_transmute_copy(&a.val),
10092 simd: self,
10093 };
10094 let rotated = int.rotate_elements_left::<OFFSET>();
10095 mask16x32 {
10096 val: crate::transmute::checked_transmute_copy(&rotated.val),
10097 simd: self,
10098 }
10099 }
10100 #[inline(always)]
10101 fn rotate_elements_right_mask16x32<const OFFSET: usize>(
10102 self,
10103 a: mask16x32<Self>,
10104 ) -> mask16x32<Self> {
10105 let int = i16x32 {
10106 val: crate::transmute::checked_transmute_copy(&a.val),
10107 simd: self,
10108 };
10109 let rotated = int.rotate_elements_right::<OFFSET>();
10110 mask16x32 {
10111 val: crate::transmute::checked_transmute_copy(&rotated.val),
10112 simd: self,
10113 }
10114 }
10115 #[inline(always)]
10116 fn split_mask16x32(self, a: mask16x32<Self>) -> (mask16x16<Self>, mask16x16<Self>) {
10117 (
10118 mask16x16 {
10119 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
10120 simd: self,
10121 },
10122 mask16x16 {
10123 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
10124 simd: self,
10125 },
10126 )
10127 }
10128 #[inline(always)]
10129 fn slide_i32x16<const SHIFT: usize>(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
10130 if SHIFT >= 16usize {
10131 return b;
10132 }
10133 let result = cross_block_alignr_128x4(
10134 self,
10135 Bytes::to_bytes(b).val.0,
10136 Bytes::to_bytes(a).val.0,
10137 SHIFT * 4usize,
10138 );
10139 Bytes::from_bytes(u8x64 {
10140 val: crate::support::Aligned512(result),
10141 simd: self,
10142 })
10143 }
10144 #[inline(always)]
10145 fn compress_i32x16(self, values: i32x16<Self>, mask: mask32x16<Self>) -> i32x16<Self> {
10146 let mask = mask8x64 {
10147 val: crate::transmute::checked_transmute_copy(&mask.val),
10148 simd: self,
10149 };
10150 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
10151 }
10152 #[inline(always)]
10153 fn compress_merge_i32x16(
10154 self,
10155 values: i32x16<Self>,
10156 mask: mask32x16<Self>,
10157 merge: i32x16<Self>,
10158 ) -> i32x16<Self> {
10159 let mask = mask8x64 {
10160 val: crate::transmute::checked_transmute_copy(&mask.val),
10161 simd: self,
10162 };
10163 Bytes::from_bytes(self.compress_merge_u8x64(
10164 Bytes::to_bytes(values),
10165 mask,
10166 Bytes::to_bytes(merge),
10167 ))
10168 }
10169 #[inline(always)]
10170 fn expand_i32x16(self, values: i32x16<Self>, mask: mask32x16<Self>) -> i32x16<Self> {
10171 let mask = mask8x64 {
10172 val: crate::transmute::checked_transmute_copy(&mask.val),
10173 simd: self,
10174 };
10175 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
10176 }
10177 #[inline(always)]
10178 fn expand_merge_i32x16(
10179 self,
10180 values: i32x16<Self>,
10181 mask: mask32x16<Self>,
10182 merge: i32x16<Self>,
10183 ) -> i32x16<Self> {
10184 let mask = mask8x64 {
10185 val: crate::transmute::checked_transmute_copy(&mask.val),
10186 simd: self,
10187 };
10188 Bytes::from_bytes(self.expand_merge_u8x64(
10189 Bytes::to_bytes(values),
10190 mask,
10191 Bytes::to_bytes(merge),
10192 ))
10193 }
10194 #[inline(always)]
10195 fn split_i32x16(self, a: i32x16<Self>) -> (i32x8<Self>, i32x8<Self>) {
10196 (
10197 i32x8 {
10198 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
10199 simd: self,
10200 },
10201 i32x8 {
10202 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
10203 simd: self,
10204 },
10205 )
10206 }
10207 #[inline(always)]
10208 fn slide_u32x16<const SHIFT: usize>(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
10209 if SHIFT >= 16usize {
10210 return b;
10211 }
10212 let result = cross_block_alignr_128x4(
10213 self,
10214 Bytes::to_bytes(b).val.0,
10215 Bytes::to_bytes(a).val.0,
10216 SHIFT * 4usize,
10217 );
10218 Bytes::from_bytes(u8x64 {
10219 val: crate::support::Aligned512(result),
10220 simd: self,
10221 })
10222 }
10223 #[inline(always)]
10224 fn compress_u32x16(self, values: u32x16<Self>, mask: mask32x16<Self>) -> u32x16<Self> {
10225 let mask = mask8x64 {
10226 val: crate::transmute::checked_transmute_copy(&mask.val),
10227 simd: self,
10228 };
10229 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
10230 }
10231 #[inline(always)]
10232 fn compress_merge_u32x16(
10233 self,
10234 values: u32x16<Self>,
10235 mask: mask32x16<Self>,
10236 merge: u32x16<Self>,
10237 ) -> u32x16<Self> {
10238 let mask = mask8x64 {
10239 val: crate::transmute::checked_transmute_copy(&mask.val),
10240 simd: self,
10241 };
10242 Bytes::from_bytes(self.compress_merge_u8x64(
10243 Bytes::to_bytes(values),
10244 mask,
10245 Bytes::to_bytes(merge),
10246 ))
10247 }
10248 #[inline(always)]
10249 fn expand_u32x16(self, values: u32x16<Self>, mask: mask32x16<Self>) -> u32x16<Self> {
10250 let mask = mask8x64 {
10251 val: crate::transmute::checked_transmute_copy(&mask.val),
10252 simd: self,
10253 };
10254 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
10255 }
10256 #[inline(always)]
10257 fn expand_merge_u32x16(
10258 self,
10259 values: u32x16<Self>,
10260 mask: mask32x16<Self>,
10261 merge: u32x16<Self>,
10262 ) -> u32x16<Self> {
10263 let mask = mask8x64 {
10264 val: crate::transmute::checked_transmute_copy(&mask.val),
10265 simd: self,
10266 };
10267 Bytes::from_bytes(self.expand_merge_u8x64(
10268 Bytes::to_bytes(values),
10269 mask,
10270 Bytes::to_bytes(merge),
10271 ))
10272 }
10273 #[inline(always)]
10274 fn split_u32x16(self, a: u32x16<Self>) -> (u32x8<Self>, u32x8<Self>) {
10275 (
10276 u32x8 {
10277 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
10278 simd: self,
10279 },
10280 u32x8 {
10281 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
10282 simd: self,
10283 },
10284 )
10285 }
10286 #[inline(always)]
10287 fn from_vector_mask32x16(self, vector: i32x16<Self>) -> mask32x16<Self> {
10288 mask32x16 {
10289 val: vector.val,
10290 simd: self,
10291 }
10292 }
10293 #[inline(always)]
10294 fn to_vector_mask32x16(self, a: mask32x16<Self>) -> i32x16<Self> {
10295 i32x16 {
10296 val: a.val,
10297 simd: self,
10298 }
10299 }
10300 #[inline(always)]
10301 fn set_mask32x16(self, a: &mut mask32x16<Self>, index: usize, value: bool) -> () {
10302 assert!(
10303 index < 16usize,
10304 "mask lane index {index} is out of bounds for {} lanes",
10305 16usize
10306 );
10307 let mut lanes: [i32; 16usize] = (*a).into();
10308 lanes[index] = if value { !0 } else { 0 };
10309 *a = lanes.simd_into(self);
10310 }
10311 #[inline(always)]
10312 fn rotate_elements_left_mask32x16<const OFFSET: usize>(
10313 self,
10314 a: mask32x16<Self>,
10315 ) -> mask32x16<Self> {
10316 let int = i32x16 {
10317 val: crate::transmute::checked_transmute_copy(&a.val),
10318 simd: self,
10319 };
10320 let rotated = int.rotate_elements_left::<OFFSET>();
10321 mask32x16 {
10322 val: crate::transmute::checked_transmute_copy(&rotated.val),
10323 simd: self,
10324 }
10325 }
10326 #[inline(always)]
10327 fn rotate_elements_right_mask32x16<const OFFSET: usize>(
10328 self,
10329 a: mask32x16<Self>,
10330 ) -> mask32x16<Self> {
10331 let int = i32x16 {
10332 val: crate::transmute::checked_transmute_copy(&a.val),
10333 simd: self,
10334 };
10335 let rotated = int.rotate_elements_right::<OFFSET>();
10336 mask32x16 {
10337 val: crate::transmute::checked_transmute_copy(&rotated.val),
10338 simd: self,
10339 }
10340 }
10341 #[inline(always)]
10342 fn split_mask32x16(self, a: mask32x16<Self>) -> (mask32x8<Self>, mask32x8<Self>) {
10343 (
10344 mask32x8 {
10345 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
10346 simd: self,
10347 },
10348 mask32x8 {
10349 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
10350 simd: self,
10351 },
10352 )
10353 }
10354 #[inline(always)]
10355 fn slide_f64x8<const SHIFT: usize>(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
10356 if SHIFT >= 8usize {
10357 return b;
10358 }
10359 let result = cross_block_alignr_128x4(
10360 self,
10361 Bytes::to_bytes(b).val.0,
10362 Bytes::to_bytes(a).val.0,
10363 SHIFT * 8usize,
10364 );
10365 Bytes::from_bytes(u8x64 {
10366 val: crate::support::Aligned512(result),
10367 simd: self,
10368 })
10369 }
10370 #[inline(always)]
10371 fn compress_f64x8(self, values: f64x8<Self>, mask: mask64x8<Self>) -> f64x8<Self> {
10372 let mask = mask8x64 {
10373 val: crate::transmute::checked_transmute_copy(&mask.val),
10374 simd: self,
10375 };
10376 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
10377 }
10378 #[inline(always)]
10379 fn compress_merge_f64x8(
10380 self,
10381 values: f64x8<Self>,
10382 mask: mask64x8<Self>,
10383 merge: f64x8<Self>,
10384 ) -> f64x8<Self> {
10385 let mask = mask8x64 {
10386 val: crate::transmute::checked_transmute_copy(&mask.val),
10387 simd: self,
10388 };
10389 Bytes::from_bytes(self.compress_merge_u8x64(
10390 Bytes::to_bytes(values),
10391 mask,
10392 Bytes::to_bytes(merge),
10393 ))
10394 }
10395 #[inline(always)]
10396 fn expand_f64x8(self, values: f64x8<Self>, mask: mask64x8<Self>) -> f64x8<Self> {
10397 let mask = mask8x64 {
10398 val: crate::transmute::checked_transmute_copy(&mask.val),
10399 simd: self,
10400 };
10401 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
10402 }
10403 #[inline(always)]
10404 fn expand_merge_f64x8(
10405 self,
10406 values: f64x8<Self>,
10407 mask: mask64x8<Self>,
10408 merge: f64x8<Self>,
10409 ) -> f64x8<Self> {
10410 let mask = mask8x64 {
10411 val: crate::transmute::checked_transmute_copy(&mask.val),
10412 simd: self,
10413 };
10414 Bytes::from_bytes(self.expand_merge_u8x64(
10415 Bytes::to_bytes(values),
10416 mask,
10417 Bytes::to_bytes(merge),
10418 ))
10419 }
10420 #[inline(always)]
10421 fn simd_ne_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> mask64x8<Self> {
10422 let (a0, a1) = self.split_f64x8(a);
10423 let (b0, b1) = self.split_f64x8(b);
10424 self.combine_mask64x4(self.simd_ne_f64x4(a0, b0), self.simd_ne_f64x4(a1, b1))
10425 }
10426 #[inline(always)]
10427 fn is_nan_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
10428 a.simd_ne(a)
10429 }
10430 #[inline(always)]
10431 fn is_infinite_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
10432 a.abs().simd_eq(f64::INFINITY)
10433 }
10434 #[inline(always)]
10435 fn is_finite_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
10436 a.abs().simd_lt(f64::INFINITY)
10437 }
10438 #[inline(always)]
10439 fn is_subnormal_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
10440 let i: i64x8<Self> = a.bitcast();
10441 (i & 0x7FF0_0000_0000_0000_i64).simd_eq(0) & (i & 0x000F_FFFF_FFFF_FFFF_i64).simd_ne(0)
10442 }
10443 #[inline(always)]
10444 fn is_normal_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
10445 let i: i64x8<Self> = a.bitcast();
10446 let exp = i & 0x7FF0_0000_0000_0000_i64;
10447 exp.simd_ne(0) & exp.simd_ne(0x7FF0_0000_0000_0000_i64)
10448 }
10449 #[inline(always)]
10450 fn is_sign_positive_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
10451 let i: i64x8<Self> = a.bitcast();
10452 (i & i64::MIN).simd_eq(0)
10453 }
10454 #[inline(always)]
10455 fn is_sign_negative_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
10456 !a.is_sign_positive()
10457 }
10458 #[inline(always)]
10459 fn split_f64x8(self, a: f64x8<Self>) -> (f64x4<Self>, f64x4<Self>) {
10460 (
10461 f64x4 {
10462 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
10463 simd: self,
10464 },
10465 f64x4 {
10466 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
10467 simd: self,
10468 },
10469 )
10470 }
10471 #[inline(always)]
10472 fn slide_i64x8<const SHIFT: usize>(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
10473 if SHIFT >= 8usize {
10474 return b;
10475 }
10476 let result = cross_block_alignr_128x4(
10477 self,
10478 Bytes::to_bytes(b).val.0,
10479 Bytes::to_bytes(a).val.0,
10480 SHIFT * 8usize,
10481 );
10482 Bytes::from_bytes(u8x64 {
10483 val: crate::support::Aligned512(result),
10484 simd: self,
10485 })
10486 }
10487 #[inline(always)]
10488 fn compress_i64x8(self, values: i64x8<Self>, mask: mask64x8<Self>) -> i64x8<Self> {
10489 let mask = mask8x64 {
10490 val: crate::transmute::checked_transmute_copy(&mask.val),
10491 simd: self,
10492 };
10493 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
10494 }
10495 #[inline(always)]
10496 fn compress_merge_i64x8(
10497 self,
10498 values: i64x8<Self>,
10499 mask: mask64x8<Self>,
10500 merge: i64x8<Self>,
10501 ) -> i64x8<Self> {
10502 let mask = mask8x64 {
10503 val: crate::transmute::checked_transmute_copy(&mask.val),
10504 simd: self,
10505 };
10506 Bytes::from_bytes(self.compress_merge_u8x64(
10507 Bytes::to_bytes(values),
10508 mask,
10509 Bytes::to_bytes(merge),
10510 ))
10511 }
10512 #[inline(always)]
10513 fn expand_i64x8(self, values: i64x8<Self>, mask: mask64x8<Self>) -> i64x8<Self> {
10514 let mask = mask8x64 {
10515 val: crate::transmute::checked_transmute_copy(&mask.val),
10516 simd: self,
10517 };
10518 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
10519 }
10520 #[inline(always)]
10521 fn expand_merge_i64x8(
10522 self,
10523 values: i64x8<Self>,
10524 mask: mask64x8<Self>,
10525 merge: i64x8<Self>,
10526 ) -> i64x8<Self> {
10527 let mask = mask8x64 {
10528 val: crate::transmute::checked_transmute_copy(&mask.val),
10529 simd: self,
10530 };
10531 Bytes::from_bytes(self.expand_merge_u8x64(
10532 Bytes::to_bytes(values),
10533 mask,
10534 Bytes::to_bytes(merge),
10535 ))
10536 }
10537 #[inline(always)]
10538 fn split_i64x8(self, a: i64x8<Self>) -> (i64x4<Self>, i64x4<Self>) {
10539 (
10540 i64x4 {
10541 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
10542 simd: self,
10543 },
10544 i64x4 {
10545 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
10546 simd: self,
10547 },
10548 )
10549 }
10550 #[inline(always)]
10551 fn slide_u64x8<const SHIFT: usize>(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
10552 if SHIFT >= 8usize {
10553 return b;
10554 }
10555 let result = cross_block_alignr_128x4(
10556 self,
10557 Bytes::to_bytes(b).val.0,
10558 Bytes::to_bytes(a).val.0,
10559 SHIFT * 8usize,
10560 );
10561 Bytes::from_bytes(u8x64 {
10562 val: crate::support::Aligned512(result),
10563 simd: self,
10564 })
10565 }
10566 #[inline(always)]
10567 fn compress_u64x8(self, values: u64x8<Self>, mask: mask64x8<Self>) -> u64x8<Self> {
10568 let mask = mask8x64 {
10569 val: crate::transmute::checked_transmute_copy(&mask.val),
10570 simd: self,
10571 };
10572 Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
10573 }
10574 #[inline(always)]
10575 fn compress_merge_u64x8(
10576 self,
10577 values: u64x8<Self>,
10578 mask: mask64x8<Self>,
10579 merge: u64x8<Self>,
10580 ) -> u64x8<Self> {
10581 let mask = mask8x64 {
10582 val: crate::transmute::checked_transmute_copy(&mask.val),
10583 simd: self,
10584 };
10585 Bytes::from_bytes(self.compress_merge_u8x64(
10586 Bytes::to_bytes(values),
10587 mask,
10588 Bytes::to_bytes(merge),
10589 ))
10590 }
10591 #[inline(always)]
10592 fn expand_u64x8(self, values: u64x8<Self>, mask: mask64x8<Self>) -> u64x8<Self> {
10593 let mask = mask8x64 {
10594 val: crate::transmute::checked_transmute_copy(&mask.val),
10595 simd: self,
10596 };
10597 Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
10598 }
10599 #[inline(always)]
10600 fn expand_merge_u64x8(
10601 self,
10602 values: u64x8<Self>,
10603 mask: mask64x8<Self>,
10604 merge: u64x8<Self>,
10605 ) -> u64x8<Self> {
10606 let mask = mask8x64 {
10607 val: crate::transmute::checked_transmute_copy(&mask.val),
10608 simd: self,
10609 };
10610 Bytes::from_bytes(self.expand_merge_u8x64(
10611 Bytes::to_bytes(values),
10612 mask,
10613 Bytes::to_bytes(merge),
10614 ))
10615 }
10616 #[inline(always)]
10617 fn split_u64x8(self, a: u64x8<Self>) -> (u64x4<Self>, u64x4<Self>) {
10618 (
10619 u64x4 {
10620 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
10621 simd: self,
10622 },
10623 u64x4 {
10624 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
10625 simd: self,
10626 },
10627 )
10628 }
10629 #[inline(always)]
10630 fn from_vector_mask64x8(self, vector: i64x8<Self>) -> mask64x8<Self> {
10631 mask64x8 {
10632 val: vector.val,
10633 simd: self,
10634 }
10635 }
10636 #[inline(always)]
10637 fn to_vector_mask64x8(self, a: mask64x8<Self>) -> i64x8<Self> {
10638 i64x8 {
10639 val: a.val,
10640 simd: self,
10641 }
10642 }
10643 #[inline(always)]
10644 fn set_mask64x8(self, a: &mut mask64x8<Self>, index: usize, value: bool) -> () {
10645 assert!(
10646 index < 8usize,
10647 "mask lane index {index} is out of bounds for {} lanes",
10648 8usize
10649 );
10650 let mut lanes: [i64; 8usize] = (*a).into();
10651 lanes[index] = if value { !0 } else { 0 };
10652 *a = lanes.simd_into(self);
10653 }
10654 #[inline(always)]
10655 fn rotate_elements_left_mask64x8<const OFFSET: usize>(
10656 self,
10657 a: mask64x8<Self>,
10658 ) -> mask64x8<Self> {
10659 let int = i64x8 {
10660 val: crate::transmute::checked_transmute_copy(&a.val),
10661 simd: self,
10662 };
10663 let rotated = int.rotate_elements_left::<OFFSET>();
10664 mask64x8 {
10665 val: crate::transmute::checked_transmute_copy(&rotated.val),
10666 simd: self,
10667 }
10668 }
10669 #[inline(always)]
10670 fn rotate_elements_right_mask64x8<const OFFSET: usize>(
10671 self,
10672 a: mask64x8<Self>,
10673 ) -> mask64x8<Self> {
10674 let int = i64x8 {
10675 val: crate::transmute::checked_transmute_copy(&a.val),
10676 simd: self,
10677 };
10678 let rotated = int.rotate_elements_right::<OFFSET>();
10679 mask64x8 {
10680 val: crate::transmute::checked_transmute_copy(&rotated.val),
10681 simd: self,
10682 }
10683 }
10684 #[inline(always)]
10685 fn split_mask64x8(self, a: mask64x8<Self>) -> (mask64x4<Self>, mask64x4<Self>) {
10686 (
10687 mask64x4 {
10688 val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
10689 simd: self,
10690 },
10691 mask64x4 {
10692 val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
10693 simd: self,
10694 },
10695 )
10696 }
10697}
10698crate::kernel!(
10699 #[doc = r" This is a version of the `alignr` intrinsic that takes a non-const shift argument. The shift is still"]
10700 #[doc = r" expected to be constant in practice, so the match statement will be optimized out. This exists because"]
10701 #[doc = r" Rust doesn't currently let you do math on const generics."]
10702 #[inline(always)]
10703 fn dyn_alignr_128(token: Sse4_2, a: __m128i, b: __m128i, shift: usize) -> __m128i {
10704 match shift {
10705 0usize => _mm_alignr_epi8::<0i32>(a, b),
10706 1usize => _mm_alignr_epi8::<1i32>(a, b),
10707 2usize => _mm_alignr_epi8::<2i32>(a, b),
10708 3usize => _mm_alignr_epi8::<3i32>(a, b),
10709 4usize => _mm_alignr_epi8::<4i32>(a, b),
10710 5usize => _mm_alignr_epi8::<5i32>(a, b),
10711 6usize => _mm_alignr_epi8::<6i32>(a, b),
10712 7usize => _mm_alignr_epi8::<7i32>(a, b),
10713 8usize => _mm_alignr_epi8::<8i32>(a, b),
10714 9usize => _mm_alignr_epi8::<9i32>(a, b),
10715 10usize => _mm_alignr_epi8::<10i32>(a, b),
10716 11usize => _mm_alignr_epi8::<11i32>(a, b),
10717 12usize => _mm_alignr_epi8::<12i32>(a, b),
10718 13usize => _mm_alignr_epi8::<13i32>(a, b),
10719 14usize => _mm_alignr_epi8::<14i32>(a, b),
10720 15usize => _mm_alignr_epi8::<15i32>(a, b),
10721 _ => unreachable!(),
10722 }
10723 }
10724);
10725crate::kernel!(
10726 #[doc = r" Concatenates `b` and `a` (each N blocks) and extracts N blocks starting at byte offset `shift_bytes`."]
10727 #[doc = r" Extracts from [b : a] (b in low bytes, a in high bytes), matching `alignr` semantics."]
10728 #[inline(always)]
10729 fn cross_block_alignr_128x2(
10730 token: Sse4_2,
10731 a: [__m128i; 2usize],
10732 b: [__m128i; 2usize],
10733 shift_bytes: usize,
10734 ) -> [__m128i; 2usize] {
10735 [
10736 {
10737 let [lo, hi] =
10738 crate::support::cross_block_slide_blocks_at(&b, &a, 0usize, shift_bytes);
10739 dyn_alignr_128(token, hi, lo, shift_bytes % 16)
10740 },
10741 {
10742 let [lo, hi] =
10743 crate::support::cross_block_slide_blocks_at(&b, &a, 1usize, shift_bytes);
10744 dyn_alignr_128(token, hi, lo, shift_bytes % 16)
10745 },
10746 ]
10747 }
10748);
10749crate::kernel!(
10750 #[doc = r" Concatenates `b` and `a` (each N blocks) and extracts N blocks starting at byte offset `shift_bytes`."]
10751 #[doc = r" Extracts from [b : a] (b in low bytes, a in high bytes), matching `alignr` semantics."]
10752 #[inline(always)]
10753 fn cross_block_alignr_128x4(
10754 token: Sse4_2,
10755 a: [__m128i; 4usize],
10756 b: [__m128i; 4usize],
10757 shift_bytes: usize,
10758 ) -> [__m128i; 4usize] {
10759 [
10760 {
10761 let [lo, hi] =
10762 crate::support::cross_block_slide_blocks_at(&b, &a, 0usize, shift_bytes);
10763 dyn_alignr_128(token, hi, lo, shift_bytes % 16)
10764 },
10765 {
10766 let [lo, hi] =
10767 crate::support::cross_block_slide_blocks_at(&b, &a, 1usize, shift_bytes);
10768 dyn_alignr_128(token, hi, lo, shift_bytes % 16)
10769 },
10770 {
10771 let [lo, hi] =
10772 crate::support::cross_block_slide_blocks_at(&b, &a, 2usize, shift_bytes);
10773 dyn_alignr_128(token, hi, lo, shift_bytes % 16)
10774 },
10775 {
10776 let [lo, hi] =
10777 crate::support::cross_block_slide_blocks_at(&b, &a, 3usize, shift_bytes);
10778 dyn_alignr_128(token, hi, lo, shift_bytes % 16)
10779 },
10780 ]
10781 }
10782);