fearless_simd/generated/
sse2.rs

1// Copyright 2025 the Fearless_SIMD Authors
2// SPDX-License-Identifier: Apache-2.0 OR MIT
3
4// This file is autogenerated by fearless_simd_gen
5
6#[cfg(all(feature = "libm", not(feature = "std")))]
7#[allow(
8    unused_imports,
9    reason = "Generated backends use different subsets of these helpers"
10)]
11use crate::math::FloatExt as _;
12use crate::{Level, arch_types::ArchTypes, prelude::*, seal::Seal};
13use crate::{
14    f32x4, f32x8, f32x16, f64x2, f64x4, f64x8, i8x16, i8x32, i8x64, i16x8, i16x16, i16x32, i32x4,
15    i32x8, i32x16, i64x2, i64x4, i64x8, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16,
16    mask16x32, mask32x4, mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64,
17    u16x8, u16x16, u16x32, u32x4, u32x8, u32x16, u64x2, u64x4, u64x8,
18};
19#[cfg(target_arch = "x86")]
20use core::arch::x86::*;
21#[cfg(target_arch = "x86_64")]
22use core::arch::x86_64::*;
23use core::ops::*;
24#[doc = "A token for SSE2 intrinsics on `x86` and `x86_64`, representing the x86-64 baseline."]
25#[doc = "# Browsing the documentation"]
26#[doc = "The method list on this struct is very verbose."]
27#[doc = "A better way to browse the docs is by looking at vector types such as [`u32x4`], [`f32x4`] or [`mask32x4`]."]
28#[doc = "They include all the operations listed here, and also provide some additional convenience methods."]
29#[derive(Clone, Copy, Debug)]
30pub struct Sse2 {
31    _private: (),
32}
33impl Sse2 {
34    #[doc = "Create a SIMD token proving that SSE2 is available.\n\nThis is the baseline on x86-64 and i686 targets. On i586 it needs runtime detection."]
35    #[doc = r""]
36    #[doc = r" Most users should safely obtain the token from [`Level::new`] instead of calling this function."]
37    #[doc = r""]
38    #[doc = r" This function can be called without an `unsafe` block from a function"]
39    #[doc = r" with all the required target features enabled via the `#[target_feature]` annotation."]
40    #[doc = r""]
41    #[doc = r" # Safety"]
42    #[doc = r""]
43    #[doc = "When invoking this function through an `unsafe` block, the caller must ensure that the current CPU supports `fxsr`, `sse`, `sse2`."]
44    #[inline]
45    #[target_feature(enable = "fxsr,sse,sse2")]
46    pub const fn assume_supported() -> Self {
47        Self { _private: () }
48    }
49}
50impl Seal for Sse2 {}
51impl ExtractToken for Sse2 {
52    type S = Self;
53    #[inline]
54    fn token(&self) -> Self {
55        *self
56    }
57}
58impl ArchTypes for Sse2 {
59    type f32x4 = crate::support::Aligned128<__m128>;
60    type i8x16 = crate::support::Aligned128<__m128i>;
61    type u8x16 = crate::support::Aligned128<__m128i>;
62    type mask8x16 = crate::support::Aligned128<__m128i>;
63    type i16x8 = crate::support::Aligned128<__m128i>;
64    type u16x8 = crate::support::Aligned128<__m128i>;
65    type mask16x8 = crate::support::Aligned128<__m128i>;
66    type i32x4 = crate::support::Aligned128<__m128i>;
67    type u32x4 = crate::support::Aligned128<__m128i>;
68    type mask32x4 = crate::support::Aligned128<__m128i>;
69    type f64x2 = crate::support::Aligned128<__m128d>;
70    type i64x2 = crate::support::Aligned128<__m128i>;
71    type u64x2 = crate::support::Aligned128<__m128i>;
72    type mask64x2 = crate::support::Aligned128<__m128i>;
73    type f32x8 = crate::support::Aligned256<[__m128; 2usize]>;
74    type i8x32 = crate::support::Aligned256<[__m128i; 2usize]>;
75    type u8x32 = crate::support::Aligned256<[__m128i; 2usize]>;
76    type mask8x32 = crate::support::Aligned256<[__m128i; 2usize]>;
77    type i16x16 = crate::support::Aligned256<[__m128i; 2usize]>;
78    type u16x16 = crate::support::Aligned256<[__m128i; 2usize]>;
79    type mask16x16 = crate::support::Aligned256<[__m128i; 2usize]>;
80    type i32x8 = crate::support::Aligned256<[__m128i; 2usize]>;
81    type u32x8 = crate::support::Aligned256<[__m128i; 2usize]>;
82    type mask32x8 = crate::support::Aligned256<[__m128i; 2usize]>;
83    type f64x4 = crate::support::Aligned256<[__m128d; 2usize]>;
84    type i64x4 = crate::support::Aligned256<[__m128i; 2usize]>;
85    type u64x4 = crate::support::Aligned256<[__m128i; 2usize]>;
86    type mask64x4 = crate::support::Aligned256<[__m128i; 2usize]>;
87    type f32x16 = crate::support::Aligned512<[__m128; 4usize]>;
88    type i8x64 = crate::support::Aligned512<[__m128i; 4usize]>;
89    type u8x64 = crate::support::Aligned512<[__m128i; 4usize]>;
90    type mask8x64 = crate::support::Aligned512<[__m128i; 4usize]>;
91    type i16x32 = crate::support::Aligned512<[__m128i; 4usize]>;
92    type u16x32 = crate::support::Aligned512<[__m128i; 4usize]>;
93    type mask16x32 = crate::support::Aligned512<[__m128i; 4usize]>;
94    type i32x16 = crate::support::Aligned512<[__m128i; 4usize]>;
95    type u32x16 = crate::support::Aligned512<[__m128i; 4usize]>;
96    type mask32x16 = crate::support::Aligned512<[__m128i; 4usize]>;
97    type f64x8 = crate::support::Aligned512<[__m128d; 4usize]>;
98    type i64x8 = crate::support::Aligned512<[__m128i; 4usize]>;
99    type u64x8 = crate::support::Aligned512<[__m128i; 4usize]>;
100    type mask64x8 = crate::support::Aligned512<[__m128i; 4usize]>;
101}
102impl Simd for Sse2 {
103    type f32s = f32x4<Self>;
104    type f64s = f64x2<Self>;
105    type u8s = u8x16<Self>;
106    type i8s = i8x16<Self>;
107    type u16s = u16x8<Self>;
108    type i16s = i16x8<Self>;
109    type u32s = u32x4<Self>;
110    type i32s = i32x4<Self>;
111    type u64s = u64x2<Self>;
112    type i64s = i64x2<Self>;
113    type mask8s = mask8x16<Self>;
114    type mask16s = mask16x8<Self>;
115    type mask32s = mask32x4<Self>;
116    type mask64s = mask64x2<Self>;
117    #[inline(always)]
118    fn level(self) -> Level {
119        Level::Sse2(self)
120    }
121    #[inline]
122    fn vectorize<F: FnOnce() -> R, R>(self, f: F) -> R {
123        #[inline]
124        #[target_feature(enable = "fxsr,sse,sse2")]
125        fn vectorize_sse2<F: FnOnce() -> R, R>(f: F) -> R {
126            f()
127        }
128        unsafe { vectorize_sse2(f) }
129    }
130    #[inline(always)]
131    fn abs_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
132        crate::kernel!(
133            #[inline(always)]
134            fn kernel(token: Sse2, a: f32x4<Sse2>) -> f32x4<Sse2> {
135                _mm_andnot_ps(_mm_set1_ps(-0.0), a.into()).simd_into(token)
136            }
137        );
138        kernel(self, a)
139    }
140    #[inline(always)]
141    fn splat_f32x4(self, val: f32) -> f32x4<Self> {
142        crate::kernel!(
143            #[inline(always)]
144            fn kernel(token: Sse2, val: f32) -> f32x4<Sse2> {
145                _mm_set1_ps(val).simd_into(token)
146            }
147        );
148        kernel(self, val)
149    }
150    #[inline(always)]
151    fn reverse_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
152        [a[3usize], a[2usize], a[1usize], a[0usize]].simd_into(self)
153    }
154    #[inline(always)]
155    fn slide_f32x4<const SHIFT: usize>(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
156        if SHIFT >= 4usize {
157            return b;
158        }
159        let result = dyn_alignr_128(
160            self,
161            Bytes::to_bytes(b).val.0,
162            Bytes::to_bytes(a).val.0,
163            SHIFT * 4usize,
164        );
165        Bytes::from_bytes(u8x16 {
166            val: crate::support::Aligned128(result),
167            simd: self,
168        })
169    }
170    #[inline(always)]
171    fn compress_f32x4(self, values: f32x4<Self>, mask: mask32x4<Self>) -> f32x4<Self> {
172        let mask = mask8x16 {
173            val: crate::transmute::checked_transmute_copy(&mask.val),
174            simd: self,
175        };
176        Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask))
177    }
178    #[inline(always)]
179    fn compress_merge_f32x4(
180        self,
181        values: f32x4<Self>,
182        mask: mask32x4<Self>,
183        merge: f32x4<Self>,
184    ) -> f32x4<Self> {
185        let mask = mask8x16 {
186            val: crate::transmute::checked_transmute_copy(&mask.val),
187            simd: self,
188        };
189        Bytes::from_bytes(self.compress_merge_u8x16(
190            Bytes::to_bytes(values),
191            mask,
192            Bytes::to_bytes(merge),
193        ))
194    }
195    #[inline(always)]
196    fn expand_f32x4(self, values: f32x4<Self>, mask: mask32x4<Self>) -> f32x4<Self> {
197        let mask = mask8x16 {
198            val: crate::transmute::checked_transmute_copy(&mask.val),
199            simd: self,
200        };
201        Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask))
202    }
203    #[inline(always)]
204    fn expand_merge_f32x4(
205        self,
206        values: f32x4<Self>,
207        mask: mask32x4<Self>,
208        merge: f32x4<Self>,
209    ) -> f32x4<Self> {
210        let mask = mask8x16 {
211            val: crate::transmute::checked_transmute_copy(&mask.val),
212            simd: self,
213        };
214        Bytes::from_bytes(self.expand_merge_u8x16(
215            Bytes::to_bytes(values),
216            mask,
217            Bytes::to_bytes(merge),
218        ))
219    }
220    #[inline(always)]
221    fn neg_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
222        crate::kernel!(
223            #[inline(always)]
224            fn kernel(token: Sse2, a: f32x4<Sse2>) -> f32x4<Sse2> {
225                _mm_xor_ps(a.into(), _mm_set1_ps(-0.0)).simd_into(token)
226            }
227        );
228        kernel(self, a)
229    }
230    #[inline(always)]
231    fn sqrt_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
232        crate::kernel!(
233            #[inline(always)]
234            fn kernel(token: Sse2, a: f32x4<Sse2>) -> f32x4<Sse2> {
235                _mm_sqrt_ps(a.into()).simd_into(token)
236            }
237        );
238        kernel(self, a)
239    }
240    #[inline(always)]
241    fn approximate_recip_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
242        crate::kernel!(
243            #[inline(always)]
244            fn kernel(token: Sse2, a: f32x4<Sse2>) -> f32x4<Sse2> {
245                _mm_rcp_ps(a.into()).simd_into(token)
246            }
247        );
248        kernel(self, a)
249    }
250    #[inline(always)]
251    fn add_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
252        crate::kernel!(
253            #[inline(always)]
254            fn kernel(token: Sse2, a: f32x4<Sse2>, b: f32x4<Sse2>) -> f32x4<Sse2> {
255                _mm_add_ps(a.into(), b.into()).simd_into(token)
256            }
257        );
258        kernel(self, a, b)
259    }
260    #[inline(always)]
261    fn sub_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
262        crate::kernel!(
263            #[inline(always)]
264            fn kernel(token: Sse2, a: f32x4<Sse2>, b: f32x4<Sse2>) -> f32x4<Sse2> {
265                _mm_sub_ps(a.into(), b.into()).simd_into(token)
266            }
267        );
268        kernel(self, a, b)
269    }
270    #[inline(always)]
271    fn mul_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
272        crate::kernel!(
273            #[inline(always)]
274            fn kernel(token: Sse2, a: f32x4<Sse2>, b: f32x4<Sse2>) -> f32x4<Sse2> {
275                _mm_mul_ps(a.into(), b.into()).simd_into(token)
276            }
277        );
278        kernel(self, a, b)
279    }
280    #[inline(always)]
281    fn div_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
282        crate::kernel!(
283            #[inline(always)]
284            fn kernel(token: Sse2, a: f32x4<Sse2>, b: f32x4<Sse2>) -> f32x4<Sse2> {
285                _mm_div_ps(a.into(), b.into()).simd_into(token)
286            }
287        );
288        kernel(self, a, b)
289    }
290    #[inline(always)]
291    fn copysign_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
292        crate::kernel!(
293            #[inline(always)]
294            fn kernel(token: Sse2, a: f32x4<Sse2>, b: f32x4<Sse2>) -> f32x4<Sse2> {
295                let mask = _mm_set1_ps(-0.0);
296                _mm_or_ps(_mm_and_ps(mask, b.into()), _mm_andnot_ps(mask, a.into()))
297                    .simd_into(token)
298            }
299        );
300        kernel(self, a, b)
301    }
302    #[inline(always)]
303    fn reduce_max_f32x4(self, a: f32x4<Self>) -> f32 {
304        crate::kernel!(
305            #[inline(always)]
306            fn kernel(token: Sse2, a: f32x4<Sse2>) -> f32 {
307                let reduced: __m128 = a.into();
308                let shifted = _mm_castsi128_ps(_mm_srli_si128::<8>(_mm_castps_si128(reduced)));
309                let reduced = { _mm_max_ps(reduced, shifted) };
310                let shifted = _mm_castsi128_ps(_mm_srli_si128::<4>(_mm_castps_si128(reduced)));
311                let reduced = { _mm_max_ps(reduced, shifted) };
312                _mm_cvtss_f32(reduced)
313            }
314        );
315        kernel(self, a)
316    }
317    #[inline(always)]
318    fn reduce_min_f32x4(self, a: f32x4<Self>) -> f32 {
319        crate::kernel!(
320            #[inline(always)]
321            fn kernel(token: Sse2, a: f32x4<Sse2>) -> f32 {
322                let reduced: __m128 = a.into();
323                let shifted = _mm_castsi128_ps(_mm_srli_si128::<8>(_mm_castps_si128(reduced)));
324                let reduced = { _mm_min_ps(reduced, shifted) };
325                let shifted = _mm_castsi128_ps(_mm_srli_si128::<4>(_mm_castps_si128(reduced)));
326                let reduced = { _mm_min_ps(reduced, shifted) };
327                _mm_cvtss_f32(reduced)
328            }
329        );
330        kernel(self, a)
331    }
332    #[inline(always)]
333    fn reduce_max_precise_f32x4(self, a: f32x4<Self>) -> f32 {
334        crate::kernel!(
335            #[inline(always)]
336            fn kernel(token: Sse2, a: f32x4<Sse2>) -> f32 {
337                let reduced: __m128 = a.into();
338                let shifted = _mm_castsi128_ps(_mm_srli_si128::<8>(_mm_castps_si128(reduced)));
339                let reduced = {
340                    let intermediate = _mm_max_ps(reduced, shifted);
341                    let shifted_is_nan = _mm_cmpunord_ps(shifted, shifted);
342                    _mm_or_ps(
343                        _mm_and_ps(shifted_is_nan, reduced),
344                        _mm_andnot_ps(shifted_is_nan, intermediate),
345                    )
346                };
347                let shifted = _mm_castsi128_ps(_mm_srli_si128::<4>(_mm_castps_si128(reduced)));
348                let reduced = {
349                    let intermediate = _mm_max_ps(reduced, shifted);
350                    let shifted_is_nan = _mm_cmpunord_ps(shifted, shifted);
351                    _mm_or_ps(
352                        _mm_and_ps(shifted_is_nan, reduced),
353                        _mm_andnot_ps(shifted_is_nan, intermediate),
354                    )
355                };
356                _mm_cvtss_f32(reduced)
357            }
358        );
359        kernel(self, a)
360    }
361    #[inline(always)]
362    fn reduce_min_precise_f32x4(self, a: f32x4<Self>) -> f32 {
363        crate::kernel!(
364            #[inline(always)]
365            fn kernel(token: Sse2, a: f32x4<Sse2>) -> f32 {
366                let reduced: __m128 = a.into();
367                let shifted = _mm_castsi128_ps(_mm_srli_si128::<8>(_mm_castps_si128(reduced)));
368                let reduced = {
369                    let intermediate = _mm_min_ps(reduced, shifted);
370                    let shifted_is_nan = _mm_cmpunord_ps(shifted, shifted);
371                    _mm_or_ps(
372                        _mm_and_ps(shifted_is_nan, reduced),
373                        _mm_andnot_ps(shifted_is_nan, intermediate),
374                    )
375                };
376                let shifted = _mm_castsi128_ps(_mm_srli_si128::<4>(_mm_castps_si128(reduced)));
377                let reduced = {
378                    let intermediate = _mm_min_ps(reduced, shifted);
379                    let shifted_is_nan = _mm_cmpunord_ps(shifted, shifted);
380                    _mm_or_ps(
381                        _mm_and_ps(shifted_is_nan, reduced),
382                        _mm_andnot_ps(shifted_is_nan, intermediate),
383                    )
384                };
385                _mm_cvtss_f32(reduced)
386            }
387        );
388        kernel(self, a)
389    }
390    #[inline(always)]
391    fn reduce_sum_f32x4(self, a: f32x4<Self>) -> f32 {
392        crate::kernel!(
393            #[inline(always)]
394            fn kernel(token: Sse2, a: f32x4<Sse2>) -> f32 {
395                let a: __m128 = a.into();
396                let adjacent = _mm_add_ps(a, _mm_shuffle_ps::<0b10_11_00_01>(a, a));
397                _mm_cvtss_f32(_mm_add_ss(adjacent, _mm_movehl_ps(adjacent, adjacent)))
398            }
399        );
400        kernel(self, a)
401    }
402    #[inline(always)]
403    fn reduce_product_f32x4(self, a: f32x4<Self>) -> f32 {
404        crate::kernel!(
405            #[inline(always)]
406            fn kernel(token: Sse2, a: f32x4<Sse2>) -> f32 {
407                let a: __m128 = a.into();
408                let adjacent = _mm_mul_ps(a, _mm_shuffle_ps::<0b10_11_00_01>(a, a));
409                _mm_cvtss_f32(_mm_mul_ss(adjacent, _mm_movehl_ps(adjacent, adjacent)))
410            }
411        );
412        kernel(self, a)
413    }
414    #[inline(always)]
415    fn max_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
416        crate::kernel!(
417            #[inline(always)]
418            fn kernel(token: Sse2, a: f32x4<Sse2>, b: f32x4<Sse2>) -> f32x4<Sse2> {
419                _mm_max_ps(a.into(), b.into()).simd_into(token)
420            }
421        );
422        kernel(self, a, b)
423    }
424    #[inline(always)]
425    fn min_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
426        crate::kernel!(
427            #[inline(always)]
428            fn kernel(token: Sse2, a: f32x4<Sse2>, b: f32x4<Sse2>) -> f32x4<Sse2> {
429                _mm_min_ps(a.into(), b.into()).simd_into(token)
430            }
431        );
432        kernel(self, a, b)
433    }
434    #[inline(always)]
435    fn max_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
436        crate::kernel!(
437            #[inline(always)]
438            fn kernel(token: Sse2, a: f32x4<Sse2>, b: f32x4<Sse2>) -> f32x4<Sse2> {
439                let a = a.into();
440                let b = b.into();
441                let intermediate = _mm_max_ps(a, b);
442                let b_is_nan = _mm_cmpunord_ps(b, b);
443                _mm_or_ps(
444                    _mm_and_ps(b_is_nan, a),
445                    _mm_andnot_ps(b_is_nan, intermediate),
446                )
447                .simd_into(token)
448            }
449        );
450        kernel(self, a, b)
451    }
452    #[inline(always)]
453    fn min_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
454        crate::kernel!(
455            #[inline(always)]
456            fn kernel(token: Sse2, a: f32x4<Sse2>, b: f32x4<Sse2>) -> f32x4<Sse2> {
457                let a = a.into();
458                let b = b.into();
459                let intermediate = _mm_min_ps(a, b);
460                let b_is_nan = _mm_cmpunord_ps(b, b);
461                _mm_or_ps(
462                    _mm_and_ps(b_is_nan, a),
463                    _mm_andnot_ps(b_is_nan, intermediate),
464                )
465                .simd_into(token)
466            }
467        );
468        kernel(self, a, b)
469    }
470    #[inline(always)]
471    fn clamp_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
472        crate::kernel!(
473            #[inline(always)]
474            fn kernel(token: Sse2, a: f32x4<Sse2>, b: f32x4<Sse2>, c: f32x4<Sse2>) -> f32x4<Sse2> {
475                debug_assert!(
476                    b.simd_le(c).all_true(),
477                    "clamp requires min <= max for all elements"
478                );
479                _mm_min_ps(c.into(), _mm_max_ps(b.into(), a.into())).simd_into(token)
480            }
481        );
482        kernel(self, a, b, c)
483    }
484    #[inline(always)]
485    fn simd_eq_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
486        crate::kernel!(
487            #[inline(always)]
488            fn kernel(token: Sse2, a: f32x4<Sse2>, b: f32x4<Sse2>) -> mask32x4<Sse2> {
489                _mm_castps_si128(_mm_cmpeq_ps(a.into(), b.into())).simd_into(token)
490            }
491        );
492        kernel(self, a, b)
493    }
494    #[inline(always)]
495    fn simd_ne_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
496        crate::kernel!(
497            #[inline(always)]
498            fn kernel(token: Sse2, a: f32x4<Sse2>, b: f32x4<Sse2>) -> mask32x4<Sse2> {
499                _mm_castps_si128(_mm_cmpneq_ps(a.into(), b.into())).simd_into(token)
500            }
501        );
502        kernel(self, a, b)
503    }
504    #[inline(always)]
505    fn simd_lt_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
506        crate::kernel!(
507            #[inline(always)]
508            fn kernel(token: Sse2, a: f32x4<Sse2>, b: f32x4<Sse2>) -> mask32x4<Sse2> {
509                _mm_castps_si128(_mm_cmplt_ps(a.into(), b.into())).simd_into(token)
510            }
511        );
512        kernel(self, a, b)
513    }
514    #[inline(always)]
515    fn simd_le_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
516        crate::kernel!(
517            #[inline(always)]
518            fn kernel(token: Sse2, a: f32x4<Sse2>, b: f32x4<Sse2>) -> mask32x4<Sse2> {
519                _mm_castps_si128(_mm_cmple_ps(a.into(), b.into())).simd_into(token)
520            }
521        );
522        kernel(self, a, b)
523    }
524    #[inline(always)]
525    fn zip_low_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
526        crate::kernel!(
527            #[inline(always)]
528            fn kernel(token: Sse2, a: f32x4<Sse2>, b: f32x4<Sse2>) -> f32x4<Sse2> {
529                _mm_unpacklo_ps(a.into(), b.into()).simd_into(token)
530            }
531        );
532        kernel(self, a, b)
533    }
534    #[inline(always)]
535    fn zip_high_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
536        crate::kernel!(
537            #[inline(always)]
538            fn kernel(token: Sse2, a: f32x4<Sse2>, b: f32x4<Sse2>) -> f32x4<Sse2> {
539                _mm_unpackhi_ps(a.into(), b.into()).simd_into(token)
540            }
541        );
542        kernel(self, a, b)
543    }
544    #[inline(always)]
545    fn unzip_low_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
546        crate::kernel!(
547            #[inline(always)]
548            fn kernel(token: Sse2, a: f32x4<Sse2>, b: f32x4<Sse2>) -> f32x4<Sse2> {
549                _mm_shuffle_ps::<0b10_00_10_00>(a.into(), b.into()).simd_into(token)
550            }
551        );
552        kernel(self, a, b)
553    }
554    #[inline(always)]
555    fn unzip_high_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
556        crate::kernel!(
557            #[inline(always)]
558            fn kernel(token: Sse2, a: f32x4<Sse2>, b: f32x4<Sse2>) -> f32x4<Sse2> {
559                _mm_shuffle_ps::<0b11_01_11_01>(a.into(), b.into()).simd_into(token)
560            }
561        );
562        kernel(self, a, b)
563    }
564    #[inline(always)]
565    fn interleave_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> (f32x4<Self>, f32x4<Self>) {
566        (self.zip_low_f32x4(a, b), self.zip_high_f32x4(a, b))
567    }
568    #[inline(always)]
569    fn deinterleave_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> (f32x4<Self>, f32x4<Self>) {
570        (self.unzip_low_f32x4(a, b), self.unzip_high_f32x4(a, b))
571    }
572    #[inline(always)]
573    fn mul_add_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
574        a * b + c
575    }
576    #[inline(always)]
577    fn mul_add_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
578        crate::math::mul_add_precise_f32x4(self, a, b, c)
579    }
580    #[inline(always)]
581    fn mul_sub_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
582        a * b - c
583    }
584    #[inline(always)]
585    fn mul_sub_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
586        self.mul_add_precise_f32x4(a, b, -c)
587    }
588    #[inline(always)]
589    fn floor_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
590        [
591            f32::floor(a[0usize]),
592            f32::floor(a[1usize]),
593            f32::floor(a[2usize]),
594            f32::floor(a[3usize]),
595        ]
596        .simd_into(self)
597    }
598    #[inline(always)]
599    fn ceil_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
600        [
601            f32::ceil(a[0usize]),
602            f32::ceil(a[1usize]),
603            f32::ceil(a[2usize]),
604            f32::ceil(a[3usize]),
605        ]
606        .simd_into(self)
607    }
608    #[inline(always)]
609    fn round_ties_even_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
610        [
611            f32::round_ties_even(a[0usize]),
612            f32::round_ties_even(a[1usize]),
613            f32::round_ties_even(a[2usize]),
614            f32::round_ties_even(a[3usize]),
615        ]
616        .simd_into(self)
617    }
618    #[inline(always)]
619    fn round_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
620        let bias = self.splat_f32x4(const { f32::next_down(0.5) });
621        (a + self.copysign_f32x4(bias, a)).trunc()
622    }
623    #[inline(always)]
624    fn fract_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
625        a - self.trunc_f32x4(a)
626    }
627    #[inline(always)]
628    fn trunc_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
629        [
630            f32::trunc(a[0usize]),
631            f32::trunc(a[1usize]),
632            f32::trunc(a[2usize]),
633            f32::trunc(a[3usize]),
634        ]
635        .simd_into(self)
636    }
637    #[inline(always)]
638    fn to_degrees_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
639        a * const { f32::to_degrees(1.0) }
640    }
641    #[inline(always)]
642    fn to_radians_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
643        a * const { f32::to_radians(1.0) }
644    }
645    #[inline(always)]
646    fn is_nan_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
647        a.simd_ne(a)
648    }
649    #[inline(always)]
650    fn is_infinite_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
651        a.abs().simd_eq(f32::INFINITY)
652    }
653    #[inline(always)]
654    fn is_finite_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
655        a.abs().simd_lt(f32::INFINITY)
656    }
657    #[inline(always)]
658    fn is_subnormal_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
659        let i: i32x4<Self> = a.bitcast();
660        (i & 0x7F80_0000_i32).simd_eq(0) & (i & 0x007F_FFFF_i32).simd_ne(0)
661    }
662    #[inline(always)]
663    fn is_normal_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
664        let i: i32x4<Self> = a.bitcast();
665        let exp = i & 0x7F80_0000_i32;
666        exp.simd_ne(0) & exp.simd_ne(0x7F80_0000_i32)
667    }
668    #[inline(always)]
669    fn is_sign_positive_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
670        let i: i32x4<Self> = a.bitcast();
671        (i & i32::MIN).simd_eq(0)
672    }
673    #[inline(always)]
674    fn is_sign_negative_f32x4(self, a: f32x4<Self>) -> mask32x4<Self> {
675        !a.is_sign_positive()
676    }
677    #[inline(always)]
678    fn select_f32x4(self, a: mask32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
679        crate::kernel!(
680            #[inline(always)]
681            fn kernel(
682                token: Sse2,
683                a: mask32x4<Sse2>,
684                b: f32x4<Sse2>,
685                c: f32x4<Sse2>,
686            ) -> f32x4<Sse2> {
687                _mm_or_ps(
688                    _mm_and_ps(_mm_castsi128_ps(a.into()), b.into()),
689                    _mm_andnot_ps(_mm_castsi128_ps(a.into()), c.into()),
690                )
691                .simd_into(token)
692            }
693        );
694        kernel(self, a, b, c)
695    }
696    #[inline(always)]
697    fn combine_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x8<Self> {
698        f32x8 {
699            val: crate::support::Aligned256([a.val.0, b.val.0]),
700            simd: self,
701        }
702    }
703    #[inline(always)]
704    fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4<Self>; 4usize] {
705        crate::kernel!(
706            #[inline(always)]
707            fn kernel(token: Sse2, src: &[f32; 16usize]) -> [f32x4<Sse2>; 4usize] {
708                let (chunks, []) = src.as_chunks::<4usize>() else {
709                    unreachable!()
710                };
711                let v0: __m128 =
712                    crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[0]);
713                let v1: __m128 =
714                    crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[1]);
715                let v2: __m128 =
716                    crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[2]);
717                let v3: __m128 =
718                    crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[3]);
719                let tmp0 = _mm_unpacklo_ps(v0, v1);
720                let tmp1 = _mm_unpackhi_ps(v0, v1);
721                let tmp2 = _mm_unpacklo_ps(v2, v3);
722                let tmp3 = _mm_unpackhi_ps(v2, v3);
723                let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2)));
724                let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2)));
725                let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3)));
726                let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3)));
727                [
728                    out0.simd_into(token),
729                    out1.simd_into(token),
730                    out2.simd_into(token),
731                    out3.simd_into(token),
732                ]
733            }
734        );
735        kernel(self, src)
736    }
737    #[inline(always)]
738    fn store_four_interleaved_f32x4(
739        self,
740        vectors: [f32x4<Self>; 4usize],
741        dest: &mut [f32; 16usize],
742    ) -> () {
743        crate::kernel!(
744            #[inline(always)]
745            fn kernel(
746                token: Sse2,
747                vectors: [f32x4<Sse2>; 4usize],
748                dest: &mut [f32; 16usize],
749            ) -> () {
750                let _ = token;
751                let v0: __m128 = vectors[0].into();
752                let v1: __m128 = vectors[1].into();
753                let v2: __m128 = vectors[2].into();
754                let v3: __m128 = vectors[3].into();
755                let tmp0 = _mm_unpacklo_ps(v0, v1);
756                let tmp1 = _mm_unpackhi_ps(v0, v1);
757                let tmp2 = _mm_unpacklo_ps(v2, v3);
758                let tmp3 = _mm_unpackhi_ps(v2, v3);
759                let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2)));
760                let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2)));
761                let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3)));
762                let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3)));
763                let (chunks, []) = dest.as_chunks_mut::<4usize>() else {
764                    unreachable!()
765                };
766                crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>(
767                    out0,
768                    &mut chunks[0],
769                );
770                crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>(
771                    out1,
772                    &mut chunks[1],
773                );
774                crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>(
775                    out2,
776                    &mut chunks[2],
777                );
778                crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>(
779                    out3,
780                    &mut chunks[3],
781                );
782            }
783        );
784        kernel(self, vectors, dest);
785    }
786    #[inline(always)]
787    fn widen_f32x4(self, a: f32x4<Self>) -> (f64x2<Self>, f64x2<Self>) {
788        crate::kernel!(
789            #[inline(always)]
790            fn kernel(token: Sse2, a: f32x4<Sse2>) -> (f64x2<Sse2>, f64x2<Sse2>) {
791                let raw = a.into();
792                (
793                    _mm_cvtps_pd(raw).simd_into(token),
794                    _mm_cvtps_pd(_mm_movehl_ps(raw, raw)).simd_into(token),
795                )
796            }
797        );
798        kernel(self, a)
799    }
800    #[inline(always)]
801    fn cvt_u32_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
802        crate::kernel!(
803            #[inline(always)]
804            fn kernel(token: Sse2, a: f32x4<Sse2>) -> u32x4<Sse2> {
805                let mut converted = _mm_cvttps_epi32(a.into());
806                let in_range = _mm_cmplt_ps(a.into(), _mm_set1_ps(2147483648.0));
807                let all_in_range = _mm_movemask_ps(in_range) == 0b1111;
808                if !all_in_range {
809                    let excess = _mm_sub_ps(a.into(), _mm_set1_ps(2147483648.0));
810                    let excess_converted = _mm_cvttps_epi32(_mm_andnot_ps(in_range, excess));
811                    converted = _mm_add_epi32(converted, excess_converted);
812                }
813                converted.simd_into(token)
814            }
815        );
816        kernel(self, a)
817    }
818    #[inline(always)]
819    fn cvt_u32_precise_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
820        crate::kernel!(
821            #[inline(always)]
822            fn kernel(token: Sse2, a: f32x4<Sse2>) -> u32x4<Sse2> {
823                let a = _mm_max_ps(a.into(), _mm_setzero_ps());
824                let mut converted = _mm_cvttps_epi32(a);
825                let in_range = _mm_cmplt_ps(a, _mm_set1_ps(2147483648.0));
826                let all_in_range = _mm_movemask_ps(in_range) == 0b1111;
827                if !all_in_range {
828                    let exceeds_unsigned_range =
829                        _mm_castps_si128(_mm_cmplt_ps(_mm_set1_ps(4294967040.0), a));
830                    let excess = _mm_sub_ps(a, _mm_set1_ps(2147483648.0));
831                    let excess_converted = _mm_cvttps_epi32(_mm_andnot_ps(in_range, excess));
832                    converted = _mm_add_epi32(converted, excess_converted);
833                    converted = _mm_or_si128(converted, exceeds_unsigned_range);
834                }
835                converted.simd_into(token)
836            }
837        );
838        kernel(self, a)
839    }
840    #[inline(always)]
841    fn cvt_i32_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
842        crate::kernel!(
843            #[inline(always)]
844            fn kernel(token: Sse2, a: f32x4<Sse2>) -> i32x4<Sse2> {
845                _mm_cvttps_epi32(a.into()).simd_into(token)
846            }
847        );
848        kernel(self, a)
849    }
850    #[inline(always)]
851    fn cvt_i32_precise_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
852        crate::kernel!(
853            #[inline(always)]
854            fn kernel(token: Sse2, a: f32x4<Sse2>) -> i32x4<Sse2> {
855                let a = a.into();
856                let converted = _mm_cvttps_epi32(a);
857                let positive_overflow =
858                    _mm_castps_si128(_mm_cmple_ps(_mm_set1_ps(2147483648.0), a));
859                let converted = _mm_xor_si128(converted, positive_overflow);
860                let is_not_nan = _mm_castps_si128(_mm_cmpord_ps(a, a));
861                _mm_and_si128(converted, is_not_nan).simd_into(token)
862            }
863        );
864        kernel(self, a)
865    }
866    #[inline(always)]
867    fn abs_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
868        [
869            i8::wrapping_abs(a[0usize]),
870            i8::wrapping_abs(a[1usize]),
871            i8::wrapping_abs(a[2usize]),
872            i8::wrapping_abs(a[3usize]),
873            i8::wrapping_abs(a[4usize]),
874            i8::wrapping_abs(a[5usize]),
875            i8::wrapping_abs(a[6usize]),
876            i8::wrapping_abs(a[7usize]),
877            i8::wrapping_abs(a[8usize]),
878            i8::wrapping_abs(a[9usize]),
879            i8::wrapping_abs(a[10usize]),
880            i8::wrapping_abs(a[11usize]),
881            i8::wrapping_abs(a[12usize]),
882            i8::wrapping_abs(a[13usize]),
883            i8::wrapping_abs(a[14usize]),
884            i8::wrapping_abs(a[15usize]),
885        ]
886        .simd_into(self)
887    }
888    #[inline(always)]
889    fn splat_i8x16(self, val: i8) -> i8x16<Self> {
890        crate::kernel!(
891            #[inline(always)]
892            fn kernel(token: Sse2, val: i8) -> i8x16<Sse2> {
893                _mm_set1_epi8(val).simd_into(token)
894            }
895        );
896        kernel(self, val)
897    }
898    #[inline(always)]
899    fn reverse_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
900        [
901            a[15usize], a[14usize], a[13usize], a[12usize], a[11usize], a[10usize], a[9usize],
902            a[8usize], a[7usize], a[6usize], a[5usize], a[4usize], a[3usize], a[2usize], a[1usize],
903            a[0usize],
904        ]
905        .simd_into(self)
906    }
907    #[inline(always)]
908    fn slide_i8x16<const SHIFT: usize>(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
909        if SHIFT >= 16usize {
910            return b;
911        }
912        let result = dyn_alignr_128(
913            self,
914            Bytes::to_bytes(b).val.0,
915            Bytes::to_bytes(a).val.0,
916            SHIFT,
917        );
918        Bytes::from_bytes(u8x16 {
919            val: crate::support::Aligned128(result),
920            simd: self,
921        })
922    }
923    #[inline(always)]
924    fn compress_i8x16(self, values: i8x16<Self>, mask: mask8x16<Self>) -> i8x16<Self> {
925        let mask = mask8x16 {
926            val: crate::transmute::checked_transmute_copy(&mask.val),
927            simd: self,
928        };
929        Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask))
930    }
931    #[inline(always)]
932    fn compress_merge_i8x16(
933        self,
934        values: i8x16<Self>,
935        mask: mask8x16<Self>,
936        merge: i8x16<Self>,
937    ) -> i8x16<Self> {
938        let mask = mask8x16 {
939            val: crate::transmute::checked_transmute_copy(&mask.val),
940            simd: self,
941        };
942        Bytes::from_bytes(self.compress_merge_u8x16(
943            Bytes::to_bytes(values),
944            mask,
945            Bytes::to_bytes(merge),
946        ))
947    }
948    #[inline(always)]
949    fn expand_i8x16(self, values: i8x16<Self>, mask: mask8x16<Self>) -> i8x16<Self> {
950        let mask = mask8x16 {
951            val: crate::transmute::checked_transmute_copy(&mask.val),
952            simd: self,
953        };
954        Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask))
955    }
956    #[inline(always)]
957    fn expand_merge_i8x16(
958        self,
959        values: i8x16<Self>,
960        mask: mask8x16<Self>,
961        merge: i8x16<Self>,
962    ) -> i8x16<Self> {
963        let mask = mask8x16 {
964            val: crate::transmute::checked_transmute_copy(&mask.val),
965            simd: self,
966        };
967        Bytes::from_bytes(self.expand_merge_u8x16(
968            Bytes::to_bytes(values),
969            mask,
970            Bytes::to_bytes(merge),
971        ))
972    }
973    #[inline(always)]
974    fn count_ones_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
975        [
976            i8::try_from(a[0usize].count_ones()).unwrap(),
977            i8::try_from(a[1usize].count_ones()).unwrap(),
978            i8::try_from(a[2usize].count_ones()).unwrap(),
979            i8::try_from(a[3usize].count_ones()).unwrap(),
980            i8::try_from(a[4usize].count_ones()).unwrap(),
981            i8::try_from(a[5usize].count_ones()).unwrap(),
982            i8::try_from(a[6usize].count_ones()).unwrap(),
983            i8::try_from(a[7usize].count_ones()).unwrap(),
984            i8::try_from(a[8usize].count_ones()).unwrap(),
985            i8::try_from(a[9usize].count_ones()).unwrap(),
986            i8::try_from(a[10usize].count_ones()).unwrap(),
987            i8::try_from(a[11usize].count_ones()).unwrap(),
988            i8::try_from(a[12usize].count_ones()).unwrap(),
989            i8::try_from(a[13usize].count_ones()).unwrap(),
990            i8::try_from(a[14usize].count_ones()).unwrap(),
991            i8::try_from(a[15usize].count_ones()).unwrap(),
992        ]
993        .simd_into(self)
994    }
995    #[inline(always)]
996    fn count_zeros_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
997        [
998            i8::try_from(a[0usize].count_zeros()).unwrap(),
999            i8::try_from(a[1usize].count_zeros()).unwrap(),
1000            i8::try_from(a[2usize].count_zeros()).unwrap(),
1001            i8::try_from(a[3usize].count_zeros()).unwrap(),
1002            i8::try_from(a[4usize].count_zeros()).unwrap(),
1003            i8::try_from(a[5usize].count_zeros()).unwrap(),
1004            i8::try_from(a[6usize].count_zeros()).unwrap(),
1005            i8::try_from(a[7usize].count_zeros()).unwrap(),
1006            i8::try_from(a[8usize].count_zeros()).unwrap(),
1007            i8::try_from(a[9usize].count_zeros()).unwrap(),
1008            i8::try_from(a[10usize].count_zeros()).unwrap(),
1009            i8::try_from(a[11usize].count_zeros()).unwrap(),
1010            i8::try_from(a[12usize].count_zeros()).unwrap(),
1011            i8::try_from(a[13usize].count_zeros()).unwrap(),
1012            i8::try_from(a[14usize].count_zeros()).unwrap(),
1013            i8::try_from(a[15usize].count_zeros()).unwrap(),
1014        ]
1015        .simd_into(self)
1016    }
1017    #[inline(always)]
1018    fn add_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1019        crate::kernel!(
1020            #[inline(always)]
1021            fn kernel(token: Sse2, a: i8x16<Sse2>, b: i8x16<Sse2>) -> i8x16<Sse2> {
1022                _mm_add_epi8(a.into(), b.into()).simd_into(token)
1023            }
1024        );
1025        kernel(self, a, b)
1026    }
1027    #[inline(always)]
1028    fn saturating_add_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1029        crate::kernel!(
1030            #[inline(always)]
1031            fn kernel(token: Sse2, a: i8x16<Sse2>, b: i8x16<Sse2>) -> i8x16<Sse2> {
1032                _mm_adds_epi8(a.into(), b.into()).simd_into(token)
1033            }
1034        );
1035        kernel(self, a, b)
1036    }
1037    #[inline(always)]
1038    fn sub_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1039        crate::kernel!(
1040            #[inline(always)]
1041            fn kernel(token: Sse2, a: i8x16<Sse2>, b: i8x16<Sse2>) -> i8x16<Sse2> {
1042                _mm_sub_epi8(a.into(), b.into()).simd_into(token)
1043            }
1044        );
1045        kernel(self, a, b)
1046    }
1047    #[inline(always)]
1048    fn saturating_sub_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1049        crate::kernel!(
1050            #[inline(always)]
1051            fn kernel(token: Sse2, a: i8x16<Sse2>, b: i8x16<Sse2>) -> i8x16<Sse2> {
1052                _mm_subs_epi8(a.into(), b.into()).simd_into(token)
1053            }
1054        );
1055        kernel(self, a, b)
1056    }
1057    #[inline(always)]
1058    fn mul_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1059        crate::kernel!(
1060            #[inline(always)]
1061            fn kernel(token: Sse2, a: i8x16<Sse2>, b: i8x16<Sse2>) -> i8x16<Sse2> {
1062                let dst_even = _mm_mullo_epi16(a.into(), b.into());
1063                let dst_odd =
1064                    _mm_mullo_epi16(_mm_srli_epi16::<8>(a.into()), _mm_srli_epi16::<8>(b.into()));
1065                _mm_or_si128(
1066                    _mm_slli_epi16(dst_odd, 8),
1067                    _mm_and_si128(dst_even, _mm_set1_epi16(0xFF)),
1068                )
1069                .simd_into(token)
1070            }
1071        );
1072        kernel(self, a, b)
1073    }
1074    #[inline(always)]
1075    fn and_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1076        crate::kernel!(
1077            #[inline(always)]
1078            fn kernel(token: Sse2, a: i8x16<Sse2>, b: i8x16<Sse2>) -> i8x16<Sse2> {
1079                _mm_and_si128(a.into(), b.into()).simd_into(token)
1080            }
1081        );
1082        kernel(self, a, b)
1083    }
1084    #[inline(always)]
1085    fn or_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1086        crate::kernel!(
1087            #[inline(always)]
1088            fn kernel(token: Sse2, a: i8x16<Sse2>, b: i8x16<Sse2>) -> i8x16<Sse2> {
1089                _mm_or_si128(a.into(), b.into()).simd_into(token)
1090            }
1091        );
1092        kernel(self, a, b)
1093    }
1094    #[inline(always)]
1095    fn xor_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1096        crate::kernel!(
1097            #[inline(always)]
1098            fn kernel(token: Sse2, a: i8x16<Sse2>, b: i8x16<Sse2>) -> i8x16<Sse2> {
1099                _mm_xor_si128(a.into(), b.into()).simd_into(token)
1100            }
1101        );
1102        kernel(self, a, b)
1103    }
1104    #[inline(always)]
1105    fn not_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
1106        a ^ !0
1107    }
1108    #[inline(always)]
1109    fn shl_i8x16(self, a: i8x16<Self>, shift: u32) -> i8x16<Self> {
1110        crate::kernel!(
1111            #[inline(always)]
1112            fn kernel(token: Sse2, a: i8x16<Sse2>, shift: u32) -> i8x16<Sse2> {
1113                let val = a.into();
1114                let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
1115                let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
1116                let byte_mask = _mm_set1_epi8(mask_byte);
1117                _mm_sll_epi16(_mm_and_si128(val, byte_mask), shift_count).simd_into(token)
1118            }
1119        );
1120        kernel(self, a, shift)
1121    }
1122    #[inline(always)]
1123    fn shlv_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1124        [
1125            i8::wrapping_shl(a[0usize], b[0usize] as u32),
1126            i8::wrapping_shl(a[1usize], b[1usize] as u32),
1127            i8::wrapping_shl(a[2usize], b[2usize] as u32),
1128            i8::wrapping_shl(a[3usize], b[3usize] as u32),
1129            i8::wrapping_shl(a[4usize], b[4usize] as u32),
1130            i8::wrapping_shl(a[5usize], b[5usize] as u32),
1131            i8::wrapping_shl(a[6usize], b[6usize] as u32),
1132            i8::wrapping_shl(a[7usize], b[7usize] as u32),
1133            i8::wrapping_shl(a[8usize], b[8usize] as u32),
1134            i8::wrapping_shl(a[9usize], b[9usize] as u32),
1135            i8::wrapping_shl(a[10usize], b[10usize] as u32),
1136            i8::wrapping_shl(a[11usize], b[11usize] as u32),
1137            i8::wrapping_shl(a[12usize], b[12usize] as u32),
1138            i8::wrapping_shl(a[13usize], b[13usize] as u32),
1139            i8::wrapping_shl(a[14usize], b[14usize] as u32),
1140            i8::wrapping_shl(a[15usize], b[15usize] as u32),
1141        ]
1142        .simd_into(self)
1143    }
1144    #[inline(always)]
1145    fn shr_i8x16(self, a: i8x16<Self>, shift: u32) -> i8x16<Self> {
1146        crate::kernel!(
1147            #[inline(always)]
1148            fn kernel(token: Sse2, a: i8x16<Sse2>, shift: u32) -> i8x16<Sse2> {
1149                let val = a.into();
1150                let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
1151                let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
1152                let byte_mask = _mm_set1_epi8(mask_byte);
1153                let shifted = _mm_srl_epi16(val, shift_count);
1154                let result = {
1155                    let sign = _mm_cmpgt_epi8(_mm_setzero_si128(), val);
1156                    _mm_or_si128(
1157                        _mm_and_si128(shifted, byte_mask),
1158                        _mm_andnot_si128(byte_mask, sign),
1159                    )
1160                };
1161                result.simd_into(token)
1162            }
1163        );
1164        kernel(self, a, shift)
1165    }
1166    #[inline(always)]
1167    fn shrv_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1168        [
1169            i8::wrapping_shr(a[0usize], b[0usize] as u32),
1170            i8::wrapping_shr(a[1usize], b[1usize] as u32),
1171            i8::wrapping_shr(a[2usize], b[2usize] as u32),
1172            i8::wrapping_shr(a[3usize], b[3usize] as u32),
1173            i8::wrapping_shr(a[4usize], b[4usize] as u32),
1174            i8::wrapping_shr(a[5usize], b[5usize] as u32),
1175            i8::wrapping_shr(a[6usize], b[6usize] as u32),
1176            i8::wrapping_shr(a[7usize], b[7usize] as u32),
1177            i8::wrapping_shr(a[8usize], b[8usize] as u32),
1178            i8::wrapping_shr(a[9usize], b[9usize] as u32),
1179            i8::wrapping_shr(a[10usize], b[10usize] as u32),
1180            i8::wrapping_shr(a[11usize], b[11usize] as u32),
1181            i8::wrapping_shr(a[12usize], b[12usize] as u32),
1182            i8::wrapping_shr(a[13usize], b[13usize] as u32),
1183            i8::wrapping_shr(a[14usize], b[14usize] as u32),
1184            i8::wrapping_shr(a[15usize], b[15usize] as u32),
1185        ]
1186        .simd_into(self)
1187    }
1188    #[inline(always)]
1189    fn reduce_max_i8x16(self, a: i8x16<Self>) -> i8 {
1190        crate::kernel!(
1191            #[inline(always)]
1192            fn kernel(token: Sse2, a: i8x16<Sse2>) -> i8 {
1193                let reduced: __m128i = a.into();
1194                let shifted = _mm_srli_si128::<8>(reduced);
1195                let reduced = {
1196                    {
1197                        let a = reduced;
1198                        let b = shifted;
1199                        let gt = _mm_cmpgt_epi8(a, b);
1200                        _mm_or_si128(_mm_and_si128(gt, a), _mm_andnot_si128(gt, b))
1201                    }
1202                };
1203                let shifted = _mm_srli_si128::<4>(reduced);
1204                let reduced = {
1205                    {
1206                        let a = reduced;
1207                        let b = shifted;
1208                        let gt = _mm_cmpgt_epi8(a, b);
1209                        _mm_or_si128(_mm_and_si128(gt, a), _mm_andnot_si128(gt, b))
1210                    }
1211                };
1212                let shifted = _mm_srli_si128::<2>(reduced);
1213                let reduced = {
1214                    {
1215                        let a = reduced;
1216                        let b = shifted;
1217                        let gt = _mm_cmpgt_epi8(a, b);
1218                        _mm_or_si128(_mm_and_si128(gt, a), _mm_andnot_si128(gt, b))
1219                    }
1220                };
1221                let shifted = _mm_srli_si128::<1>(reduced);
1222                let reduced = {
1223                    {
1224                        let a = reduced;
1225                        let b = shifted;
1226                        let gt = _mm_cmpgt_epi8(a, b);
1227                        _mm_or_si128(_mm_and_si128(gt, a), _mm_andnot_si128(gt, b))
1228                    }
1229                };
1230                {
1231                    let lanes: [i8; 16usize] = crate::transmute::checked_transmute_copy(&reduced);
1232                    lanes[0]
1233                }
1234            }
1235        );
1236        kernel(self, a)
1237    }
1238    #[inline(always)]
1239    fn reduce_min_i8x16(self, a: i8x16<Self>) -> i8 {
1240        crate::kernel!(
1241            #[inline(always)]
1242            fn kernel(token: Sse2, a: i8x16<Sse2>) -> i8 {
1243                let reduced: __m128i = a.into();
1244                let shifted = _mm_srli_si128::<8>(reduced);
1245                let reduced = {
1246                    {
1247                        let a = reduced;
1248                        let b = shifted;
1249                        let gt = _mm_cmpgt_epi8(a, b);
1250                        _mm_or_si128(_mm_and_si128(gt, b), _mm_andnot_si128(gt, a))
1251                    }
1252                };
1253                let shifted = _mm_srli_si128::<4>(reduced);
1254                let reduced = {
1255                    {
1256                        let a = reduced;
1257                        let b = shifted;
1258                        let gt = _mm_cmpgt_epi8(a, b);
1259                        _mm_or_si128(_mm_and_si128(gt, b), _mm_andnot_si128(gt, a))
1260                    }
1261                };
1262                let shifted = _mm_srli_si128::<2>(reduced);
1263                let reduced = {
1264                    {
1265                        let a = reduced;
1266                        let b = shifted;
1267                        let gt = _mm_cmpgt_epi8(a, b);
1268                        _mm_or_si128(_mm_and_si128(gt, b), _mm_andnot_si128(gt, a))
1269                    }
1270                };
1271                let shifted = _mm_srli_si128::<1>(reduced);
1272                let reduced = {
1273                    {
1274                        let a = reduced;
1275                        let b = shifted;
1276                        let gt = _mm_cmpgt_epi8(a, b);
1277                        _mm_or_si128(_mm_and_si128(gt, b), _mm_andnot_si128(gt, a))
1278                    }
1279                };
1280                {
1281                    let lanes: [i8; 16usize] = crate::transmute::checked_transmute_copy(&reduced);
1282                    lanes[0]
1283                }
1284            }
1285        );
1286        kernel(self, a)
1287    }
1288    #[inline(always)]
1289    fn reduce_sum_i8x16(self, a: i8x16<Self>) -> i8 {
1290        crate::kernel!(
1291            #[inline(always)]
1292            fn kernel(token: Sse2, a: i8x16<Sse2>) -> i8 {
1293                let sum: __m128i = a.into();
1294                let sum = _mm_add_epi8(sum, _mm_srli_si128::<8>(sum));
1295                let sum = _mm_add_epi8(sum, _mm_srli_si128::<4>(sum));
1296                let sum = _mm_add_epi8(sum, _mm_srli_si128::<2>(sum));
1297                let sum = _mm_add_epi8(sum, _mm_srli_si128::<1>(sum));
1298                let lanes: [i8; 16usize] = crate::transmute::checked_transmute_copy(&sum);
1299                lanes[0]
1300            }
1301        );
1302        kernel(self, a)
1303    }
1304    #[inline(always)]
1305    fn reduce_product_i8x16(self, a: i8x16<Self>) -> i8 {
1306        crate::kernel!(
1307            #[inline(always)]
1308            fn kernel(token: Sse2, a: i8x16<Sse2>) -> i8 {
1309                let value: __m128i = a.into();
1310                let product = _mm_mullo_epi16(
1311                    _mm_unpacklo_epi8(value, value),
1312                    _mm_unpackhi_epi8(value, value),
1313                );
1314                let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b11_10_11_10>(product));
1315                let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b01_01_01_01>(product));
1316                let product = _mm_mullo_epi16(product, _mm_srli_epi32::<16>(product));
1317                let lanes: [i8; 16usize] = crate::transmute::checked_transmute_copy(&product);
1318                lanes[0]
1319            }
1320        );
1321        kernel(self, a)
1322    }
1323    #[inline(always)]
1324    fn max_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1325        crate::kernel!(
1326            #[inline(always)]
1327            fn kernel(token: Sse2, a: i8x16<Sse2>, b: i8x16<Sse2>) -> i8x16<Sse2> {
1328                {
1329                    let a = a.into();
1330                    let b = b.into();
1331                    let gt = _mm_cmpgt_epi8(a, b);
1332                    _mm_or_si128(_mm_and_si128(gt, a), _mm_andnot_si128(gt, b))
1333                }
1334                .simd_into(token)
1335            }
1336        );
1337        kernel(self, a, b)
1338    }
1339    #[inline(always)]
1340    fn min_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1341        crate::kernel!(
1342            #[inline(always)]
1343            fn kernel(token: Sse2, a: i8x16<Sse2>, b: i8x16<Sse2>) -> i8x16<Sse2> {
1344                {
1345                    let a = a.into();
1346                    let b = b.into();
1347                    let gt = _mm_cmpgt_epi8(a, b);
1348                    _mm_or_si128(_mm_and_si128(gt, b), _mm_andnot_si128(gt, a))
1349                }
1350                .simd_into(token)
1351            }
1352        );
1353        kernel(self, a, b)
1354    }
1355    #[inline(always)]
1356    fn clamp_i8x16(self, a: i8x16<Self>, b: i8x16<Self>, c: i8x16<Self>) -> i8x16<Self> {
1357        debug_assert!(
1358            b.simd_le(c).all_true(),
1359            "clamp requires min <= max for all elements"
1360        );
1361        a.max(b).min(c)
1362    }
1363    #[inline(always)]
1364    fn simd_eq_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
1365        crate::kernel!(
1366            #[inline(always)]
1367            fn kernel(token: Sse2, a: i8x16<Sse2>, b: i8x16<Sse2>) -> mask8x16<Sse2> {
1368                _mm_cmpeq_epi8(a.into(), b.into()).simd_into(token)
1369            }
1370        );
1371        kernel(self, a, b)
1372    }
1373    #[inline(always)]
1374    fn simd_lt_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
1375        crate::kernel!(
1376            #[inline(always)]
1377            fn kernel(token: Sse2, a: i8x16<Sse2>, b: i8x16<Sse2>) -> mask8x16<Sse2> {
1378                _mm_cmpgt_epi8(b.into(), a.into()).simd_into(token)
1379            }
1380        );
1381        kernel(self, a, b)
1382    }
1383    #[inline(always)]
1384    fn simd_le_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
1385        crate::kernel!(
1386            #[inline(always)]
1387            fn kernel(token: Sse2, a: i8x16<Sse2>, b: i8x16<Sse2>) -> mask8x16<Sse2> {
1388                {
1389                    let all_ones = _mm_set1_epi8(-1);
1390                    _mm_xor_si128(_mm_cmpgt_epi8(a.into(), b.into()), all_ones)
1391                }
1392                .simd_into(token)
1393            }
1394        );
1395        kernel(self, a, b)
1396    }
1397    #[inline(always)]
1398    fn zip_low_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1399        crate::kernel!(
1400            #[inline(always)]
1401            fn kernel(token: Sse2, a: i8x16<Sse2>, b: i8x16<Sse2>) -> i8x16<Sse2> {
1402                _mm_unpacklo_epi8(a.into(), b.into()).simd_into(token)
1403            }
1404        );
1405        kernel(self, a, b)
1406    }
1407    #[inline(always)]
1408    fn zip_high_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1409        crate::kernel!(
1410            #[inline(always)]
1411            fn kernel(token: Sse2, a: i8x16<Sse2>, b: i8x16<Sse2>) -> i8x16<Sse2> {
1412                _mm_unpackhi_epi8(a.into(), b.into()).simd_into(token)
1413            }
1414        );
1415        kernel(self, a, b)
1416    }
1417    #[inline(always)]
1418    fn unzip_low_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1419        [
1420            a[0usize], a[2usize], a[4usize], a[6usize], a[8usize], a[10usize], a[12usize],
1421            a[14usize], b[0usize], b[2usize], b[4usize], b[6usize], b[8usize], b[10usize],
1422            b[12usize], b[14usize],
1423        ]
1424        .simd_into(self)
1425    }
1426    #[inline(always)]
1427    fn unzip_high_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
1428        [
1429            a[1usize], a[3usize], a[5usize], a[7usize], a[9usize], a[11usize], a[13usize],
1430            a[15usize], b[1usize], b[3usize], b[5usize], b[7usize], b[9usize], b[11usize],
1431            b[13usize], b[15usize],
1432        ]
1433        .simd_into(self)
1434    }
1435    #[inline(always)]
1436    fn interleave_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i8x16<Self>, i8x16<Self>) {
1437        (self.zip_low_i8x16(a, b), self.zip_high_i8x16(a, b))
1438    }
1439    #[inline(always)]
1440    fn deinterleave_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i8x16<Self>, i8x16<Self>) {
1441        (self.unzip_low_i8x16(a, b), self.unzip_high_i8x16(a, b))
1442    }
1443    #[inline(always)]
1444    fn select_i8x16(self, a: mask8x16<Self>, b: i8x16<Self>, c: i8x16<Self>) -> i8x16<Self> {
1445        crate::kernel!(
1446            #[inline(always)]
1447            fn kernel(
1448                token: Sse2,
1449                a: mask8x16<Sse2>,
1450                b: i8x16<Sse2>,
1451                c: i8x16<Sse2>,
1452            ) -> i8x16<Sse2> {
1453                _mm_or_si128(
1454                    _mm_and_si128(a.into(), b.into()),
1455                    _mm_andnot_si128(a.into(), c.into()),
1456                )
1457                .simd_into(token)
1458            }
1459        );
1460        kernel(self, a, b, c)
1461    }
1462    #[inline(always)]
1463    fn combine_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x32<Self> {
1464        i8x32 {
1465            val: crate::support::Aligned256([a.val.0, b.val.0]),
1466            simd: self,
1467        }
1468    }
1469    #[inline(always)]
1470    fn neg_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
1471        crate::kernel!(
1472            #[inline(always)]
1473            fn kernel(token: Sse2, a: i8x16<Sse2>) -> i8x16<Sse2> {
1474                _mm_sub_epi8(_mm_setzero_si128(), a.into()).simd_into(token)
1475            }
1476        );
1477        kernel(self, a)
1478    }
1479    #[inline(always)]
1480    fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16<Self>; 4usize] {
1481        [
1482            [
1483                src[0usize],
1484                src[4usize],
1485                src[8usize],
1486                src[12usize],
1487                src[16usize],
1488                src[20usize],
1489                src[24usize],
1490                src[28usize],
1491                src[32usize],
1492                src[36usize],
1493                src[40usize],
1494                src[44usize],
1495                src[48usize],
1496                src[52usize],
1497                src[56usize],
1498                src[60usize],
1499            ]
1500            .simd_into(self),
1501            [
1502                src[1usize],
1503                src[5usize],
1504                src[9usize],
1505                src[13usize],
1506                src[17usize],
1507                src[21usize],
1508                src[25usize],
1509                src[29usize],
1510                src[33usize],
1511                src[37usize],
1512                src[41usize],
1513                src[45usize],
1514                src[49usize],
1515                src[53usize],
1516                src[57usize],
1517                src[61usize],
1518            ]
1519            .simd_into(self),
1520            [
1521                src[2usize],
1522                src[6usize],
1523                src[10usize],
1524                src[14usize],
1525                src[18usize],
1526                src[22usize],
1527                src[26usize],
1528                src[30usize],
1529                src[34usize],
1530                src[38usize],
1531                src[42usize],
1532                src[46usize],
1533                src[50usize],
1534                src[54usize],
1535                src[58usize],
1536                src[62usize],
1537            ]
1538            .simd_into(self),
1539            [
1540                src[3usize],
1541                src[7usize],
1542                src[11usize],
1543                src[15usize],
1544                src[19usize],
1545                src[23usize],
1546                src[27usize],
1547                src[31usize],
1548                src[35usize],
1549                src[39usize],
1550                src[43usize],
1551                src[47usize],
1552                src[51usize],
1553                src[55usize],
1554                src[59usize],
1555                src[63usize],
1556            ]
1557            .simd_into(self),
1558        ]
1559    }
1560    #[inline(always)]
1561    fn store_four_interleaved_i8x16(
1562        self,
1563        vectors: [i8x16<Self>; 4usize],
1564        dest: &mut [i8; 64usize],
1565    ) -> () {
1566        *dest = [
1567            vectors[0usize][0usize],
1568            vectors[1usize][0usize],
1569            vectors[2usize][0usize],
1570            vectors[3usize][0usize],
1571            vectors[0usize][1usize],
1572            vectors[1usize][1usize],
1573            vectors[2usize][1usize],
1574            vectors[3usize][1usize],
1575            vectors[0usize][2usize],
1576            vectors[1usize][2usize],
1577            vectors[2usize][2usize],
1578            vectors[3usize][2usize],
1579            vectors[0usize][3usize],
1580            vectors[1usize][3usize],
1581            vectors[2usize][3usize],
1582            vectors[3usize][3usize],
1583            vectors[0usize][4usize],
1584            vectors[1usize][4usize],
1585            vectors[2usize][4usize],
1586            vectors[3usize][4usize],
1587            vectors[0usize][5usize],
1588            vectors[1usize][5usize],
1589            vectors[2usize][5usize],
1590            vectors[3usize][5usize],
1591            vectors[0usize][6usize],
1592            vectors[1usize][6usize],
1593            vectors[2usize][6usize],
1594            vectors[3usize][6usize],
1595            vectors[0usize][7usize],
1596            vectors[1usize][7usize],
1597            vectors[2usize][7usize],
1598            vectors[3usize][7usize],
1599            vectors[0usize][8usize],
1600            vectors[1usize][8usize],
1601            vectors[2usize][8usize],
1602            vectors[3usize][8usize],
1603            vectors[0usize][9usize],
1604            vectors[1usize][9usize],
1605            vectors[2usize][9usize],
1606            vectors[3usize][9usize],
1607            vectors[0usize][10usize],
1608            vectors[1usize][10usize],
1609            vectors[2usize][10usize],
1610            vectors[3usize][10usize],
1611            vectors[0usize][11usize],
1612            vectors[1usize][11usize],
1613            vectors[2usize][11usize],
1614            vectors[3usize][11usize],
1615            vectors[0usize][12usize],
1616            vectors[1usize][12usize],
1617            vectors[2usize][12usize],
1618            vectors[3usize][12usize],
1619            vectors[0usize][13usize],
1620            vectors[1usize][13usize],
1621            vectors[2usize][13usize],
1622            vectors[3usize][13usize],
1623            vectors[0usize][14usize],
1624            vectors[1usize][14usize],
1625            vectors[2usize][14usize],
1626            vectors[3usize][14usize],
1627            vectors[0usize][15usize],
1628            vectors[1usize][15usize],
1629            vectors[2usize][15usize],
1630            vectors[3usize][15usize],
1631        ];
1632    }
1633    #[inline(always)]
1634    fn widen_i8x16(self, a: i8x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
1635        crate::kernel!(
1636            #[inline(always)]
1637            fn kernel(token: Sse2, a: i8x16<Sse2>) -> (i16x8<Sse2>, i16x8<Sse2>) {
1638                let raw = a.into();
1639                let sign = _mm_cmpgt_epi8(_mm_setzero_si128(), raw);
1640                (
1641                    _mm_unpacklo_epi8(raw, sign).simd_into(token),
1642                    _mm_unpackhi_epi8(raw, sign).simd_into(token),
1643                )
1644            }
1645        );
1646        kernel(self, a)
1647    }
1648    #[inline(always)]
1649    fn widening_mul_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
1650        let (a0, a1) = self.widen_i8x16(a);
1651        let (b0, b1) = self.widen_i8x16(b);
1652        (self.mul_i16x8(a0, b0), self.mul_i16x8(a1, b1))
1653    }
1654    #[inline(always)]
1655    fn widening_add_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
1656        let (a0, a1) = self.widen_i8x16(a);
1657        let (b0, b1) = self.widen_i8x16(b);
1658        (self.add_i16x8(a0, b0), self.add_i16x8(a1, b1))
1659    }
1660    #[inline(always)]
1661    fn pairwise_widening_add_i8x16(self, a: i8x16<Self>) -> i16x8<Self> {
1662        [
1663            (a[0usize] as i16) + (a[1usize] as i16),
1664            (a[2usize] as i16) + (a[3usize] as i16),
1665            (a[4usize] as i16) + (a[5usize] as i16),
1666            (a[6usize] as i16) + (a[7usize] as i16),
1667            (a[8usize] as i16) + (a[9usize] as i16),
1668            (a[10usize] as i16) + (a[11usize] as i16),
1669            (a[12usize] as i16) + (a[13usize] as i16),
1670            (a[14usize] as i16) + (a[15usize] as i16),
1671        ]
1672        .simd_into(self)
1673    }
1674    #[inline(always)]
1675    fn pairwise_dot_product_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i16x8<Self> {
1676        let (low, high) = self.widening_mul_i8x16(a, b);
1677        let (even, odd) = self.deinterleave_i16x8(low, high);
1678        self.add_i16x8(even, odd)
1679    }
1680    #[inline(always)]
1681    fn splat_u8x16(self, val: u8) -> u8x16<Self> {
1682        crate::kernel!(
1683            #[inline(always)]
1684            fn kernel(token: Sse2, val: u8) -> u8x16<Sse2> {
1685                _mm_set1_epi8(val.cast_signed()).simd_into(token)
1686            }
1687        );
1688        kernel(self, val)
1689    }
1690    #[inline(always)]
1691    fn reverse_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
1692        [
1693            a[15usize], a[14usize], a[13usize], a[12usize], a[11usize], a[10usize], a[9usize],
1694            a[8usize], a[7usize], a[6usize], a[5usize], a[4usize], a[3usize], a[2usize], a[1usize],
1695            a[0usize],
1696        ]
1697        .simd_into(self)
1698    }
1699    #[inline(always)]
1700    fn slide_u8x16<const SHIFT: usize>(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1701        if SHIFT >= 16usize {
1702            return b;
1703        }
1704        let result = dyn_alignr_128(
1705            self,
1706            Bytes::to_bytes(b).val.0,
1707            Bytes::to_bytes(a).val.0,
1708            SHIFT,
1709        );
1710        Bytes::from_bytes(u8x16 {
1711            val: crate::support::Aligned128(result),
1712            simd: self,
1713        })
1714    }
1715    #[inline(always)]
1716    fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
1717        let bytes = Bytes::to_bytes(a);
1718        let result: u8x16<Self> = [
1719            {
1720                let index = indices[0usize] as usize;
1721                bytes[index % 16usize]
1722            },
1723            {
1724                let index = indices[1usize] as usize;
1725                bytes[index % 16usize]
1726            },
1727            {
1728                let index = indices[2usize] as usize;
1729                bytes[index % 16usize]
1730            },
1731            {
1732                let index = indices[3usize] as usize;
1733                bytes[index % 16usize]
1734            },
1735            {
1736                let index = indices[4usize] as usize;
1737                bytes[index % 16usize]
1738            },
1739            {
1740                let index = indices[5usize] as usize;
1741                bytes[index % 16usize]
1742            },
1743            {
1744                let index = indices[6usize] as usize;
1745                bytes[index % 16usize]
1746            },
1747            {
1748                let index = indices[7usize] as usize;
1749                bytes[index % 16usize]
1750            },
1751            {
1752                let index = indices[8usize] as usize;
1753                bytes[index % 16usize]
1754            },
1755            {
1756                let index = indices[9usize] as usize;
1757                bytes[index % 16usize]
1758            },
1759            {
1760                let index = indices[10usize] as usize;
1761                bytes[index % 16usize]
1762            },
1763            {
1764                let index = indices[11usize] as usize;
1765                bytes[index % 16usize]
1766            },
1767            {
1768                let index = indices[12usize] as usize;
1769                bytes[index % 16usize]
1770            },
1771            {
1772                let index = indices[13usize] as usize;
1773                bytes[index % 16usize]
1774            },
1775            {
1776                let index = indices[14usize] as usize;
1777                bytes[index % 16usize]
1778            },
1779            {
1780                let index = indices[15usize] as usize;
1781                bytes[index % 16usize]
1782            },
1783        ]
1784        .simd_into(self);
1785        Bytes::from_bytes(result)
1786    }
1787    #[inline(always)]
1788    fn swizzle_dyn_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
1789        let bytes = Bytes::to_bytes(a);
1790        let mut output = [0u8; 16usize];
1791        for lane in 0..16usize {
1792            let index = indices[lane] as usize;
1793            output[lane] = bytes[index % 16usize];
1794        }
1795        let result: u8x16<Self> = output.simd_into(self);
1796        Bytes::from_bytes(result)
1797    }
1798    #[inline(always)]
1799    fn swizzle_dyn_precise_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
1800        let bytes = Bytes::to_bytes(a);
1801        let mut output = [0u8; 16usize];
1802        for lane in 0..16usize {
1803            let index = indices[lane] as usize;
1804            let value = bytes[index % 16usize];
1805            output[lane] = core::hint::select_unpredictable(index < 16usize, value, 0u8);
1806        }
1807        let result: u8x16<Self> = output.simd_into(self);
1808        Bytes::from_bytes(result)
1809    }
1810    #[inline(always)]
1811    fn concat_swizzle_dyn_u8x16(
1812        self,
1813        a: u8x16<Self>,
1814        b: u8x16<Self>,
1815        indices: u8x16<Self>,
1816    ) -> u8x16<Self> {
1817        let first_table = Bytes::to_bytes(a);
1818        let second_table = Bytes::to_bytes(b);
1819        let mut output = [0u8; 16usize];
1820        for lane in 0..16usize {
1821            let index = indices[lane] as usize % 32usize;
1822            output[lane] = if index < 16usize {
1823                first_table[index]
1824            } else {
1825                second_table[index - 16usize]
1826            };
1827        }
1828        let result: u8x16<Self> = output.simd_into(self);
1829        Bytes::from_bytes(result)
1830    }
1831    #[inline(always)]
1832    fn concat_swizzle_dyn_precise_u8x16(
1833        self,
1834        a: u8x16<Self>,
1835        b: u8x16<Self>,
1836        indices: u8x16<Self>,
1837    ) -> u8x16<Self> {
1838        let first_table = Bytes::to_bytes(a);
1839        let second_table = Bytes::to_bytes(b);
1840        let mut output = [0u8; 16usize];
1841        for lane in 0..16usize {
1842            let index = indices[lane] as usize;
1843            let table_index = index % 16usize;
1844            let value = if index < 16usize {
1845                first_table[table_index]
1846            } else {
1847                second_table[table_index]
1848            };
1849            output[lane] = if index < 32usize { value } else { 0 };
1850        }
1851        let result: u8x16<Self> = output.simd_into(self);
1852        Bytes::from_bytes(result)
1853    }
1854    #[inline(always)]
1855    fn compress_u8x16(self, values: u8x16<Self>, mask: mask8x16<Self>) -> u8x16<Self> {
1856        self.compress_merge_u8x16(values, mask, u8x16::splat(self, 0))
1857    }
1858    #[inline(always)]
1859    fn compress_merge_u8x16(
1860        self,
1861        values: u8x16<Self>,
1862        mask: mask8x16<Self>,
1863        merge: u8x16<Self>,
1864    ) -> u8x16<Self> {
1865        let mask = self.to_bitmask_mask8x16(mask);
1866        let mut merge_padded = [0u8; 16 + 1];
1867        merge_padded[..16].copy_from_slice(&*merge);
1868        let mut compacted = merge_padded;
1869        let mut output_lane = 0;
1870        for input_lane in 0..16 {
1871            compacted[output_lane] = values[input_lane];
1872            output_lane += ((mask >> input_lane) & 1) as usize;
1873        }
1874        compacted[output_lane] = merge_padded[output_lane];
1875        let mut result = merge;
1876        result.copy_from_slice(&compacted[..16]);
1877        result
1878    }
1879    #[inline(always)]
1880    fn expand_u8x16(self, values: u8x16<Self>, mask: mask8x16<Self>) -> u8x16<Self> {
1881        self.expand_merge_u8x16(values, mask, u8x16::splat(self, 0))
1882    }
1883    #[inline(always)]
1884    fn expand_merge_u8x16(
1885        self,
1886        values: u8x16<Self>,
1887        mask: mask8x16<Self>,
1888        merge: u8x16<Self>,
1889    ) -> u8x16<Self> {
1890        let mask = self.to_bitmask_mask8x16(mask);
1891        let mut result = merge;
1892        let mut input_lane = 0;
1893        for output_lane in 0..16 {
1894            let bit = ((mask >> output_lane) & 1) as u8;
1895            let keep = 0u8.wrapping_sub(bit);
1896            result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep);
1897            input_lane += usize::from(bit);
1898        }
1899        result
1900    }
1901    #[inline(always)]
1902    fn count_ones_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
1903        [
1904            u8::try_from(a[0usize].count_ones()).unwrap(),
1905            u8::try_from(a[1usize].count_ones()).unwrap(),
1906            u8::try_from(a[2usize].count_ones()).unwrap(),
1907            u8::try_from(a[3usize].count_ones()).unwrap(),
1908            u8::try_from(a[4usize].count_ones()).unwrap(),
1909            u8::try_from(a[5usize].count_ones()).unwrap(),
1910            u8::try_from(a[6usize].count_ones()).unwrap(),
1911            u8::try_from(a[7usize].count_ones()).unwrap(),
1912            u8::try_from(a[8usize].count_ones()).unwrap(),
1913            u8::try_from(a[9usize].count_ones()).unwrap(),
1914            u8::try_from(a[10usize].count_ones()).unwrap(),
1915            u8::try_from(a[11usize].count_ones()).unwrap(),
1916            u8::try_from(a[12usize].count_ones()).unwrap(),
1917            u8::try_from(a[13usize].count_ones()).unwrap(),
1918            u8::try_from(a[14usize].count_ones()).unwrap(),
1919            u8::try_from(a[15usize].count_ones()).unwrap(),
1920        ]
1921        .simd_into(self)
1922    }
1923    #[inline(always)]
1924    fn count_zeros_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
1925        [
1926            u8::try_from(a[0usize].count_zeros()).unwrap(),
1927            u8::try_from(a[1usize].count_zeros()).unwrap(),
1928            u8::try_from(a[2usize].count_zeros()).unwrap(),
1929            u8::try_from(a[3usize].count_zeros()).unwrap(),
1930            u8::try_from(a[4usize].count_zeros()).unwrap(),
1931            u8::try_from(a[5usize].count_zeros()).unwrap(),
1932            u8::try_from(a[6usize].count_zeros()).unwrap(),
1933            u8::try_from(a[7usize].count_zeros()).unwrap(),
1934            u8::try_from(a[8usize].count_zeros()).unwrap(),
1935            u8::try_from(a[9usize].count_zeros()).unwrap(),
1936            u8::try_from(a[10usize].count_zeros()).unwrap(),
1937            u8::try_from(a[11usize].count_zeros()).unwrap(),
1938            u8::try_from(a[12usize].count_zeros()).unwrap(),
1939            u8::try_from(a[13usize].count_zeros()).unwrap(),
1940            u8::try_from(a[14usize].count_zeros()).unwrap(),
1941            u8::try_from(a[15usize].count_zeros()).unwrap(),
1942        ]
1943        .simd_into(self)
1944    }
1945    #[inline(always)]
1946    fn add_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1947        crate::kernel!(
1948            #[inline(always)]
1949            fn kernel(token: Sse2, a: u8x16<Sse2>, b: u8x16<Sse2>) -> u8x16<Sse2> {
1950                _mm_add_epi8(a.into(), b.into()).simd_into(token)
1951            }
1952        );
1953        kernel(self, a, b)
1954    }
1955    #[inline(always)]
1956    fn saturating_add_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1957        crate::kernel!(
1958            #[inline(always)]
1959            fn kernel(token: Sse2, a: u8x16<Sse2>, b: u8x16<Sse2>) -> u8x16<Sse2> {
1960                _mm_adds_epu8(a.into(), b.into()).simd_into(token)
1961            }
1962        );
1963        kernel(self, a, b)
1964    }
1965    #[inline(always)]
1966    fn sub_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1967        crate::kernel!(
1968            #[inline(always)]
1969            fn kernel(token: Sse2, a: u8x16<Sse2>, b: u8x16<Sse2>) -> u8x16<Sse2> {
1970                _mm_sub_epi8(a.into(), b.into()).simd_into(token)
1971            }
1972        );
1973        kernel(self, a, b)
1974    }
1975    #[inline(always)]
1976    fn saturating_sub_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1977        crate::kernel!(
1978            #[inline(always)]
1979            fn kernel(token: Sse2, a: u8x16<Sse2>, b: u8x16<Sse2>) -> u8x16<Sse2> {
1980                _mm_subs_epu8(a.into(), b.into()).simd_into(token)
1981            }
1982        );
1983        kernel(self, a, b)
1984    }
1985    #[inline(always)]
1986    fn mul_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
1987        crate::kernel!(
1988            #[inline(always)]
1989            fn kernel(token: Sse2, a: u8x16<Sse2>, b: u8x16<Sse2>) -> u8x16<Sse2> {
1990                let dst_even = _mm_mullo_epi16(a.into(), b.into());
1991                let dst_odd =
1992                    _mm_mullo_epi16(_mm_srli_epi16::<8>(a.into()), _mm_srli_epi16::<8>(b.into()));
1993                _mm_or_si128(
1994                    _mm_slli_epi16(dst_odd, 8),
1995                    _mm_and_si128(dst_even, _mm_set1_epi16(0xFF)),
1996                )
1997                .simd_into(token)
1998            }
1999        );
2000        kernel(self, a, b)
2001    }
2002    #[inline(always)]
2003    fn and_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2004        crate::kernel!(
2005            #[inline(always)]
2006            fn kernel(token: Sse2, a: u8x16<Sse2>, b: u8x16<Sse2>) -> u8x16<Sse2> {
2007                _mm_and_si128(a.into(), b.into()).simd_into(token)
2008            }
2009        );
2010        kernel(self, a, b)
2011    }
2012    #[inline(always)]
2013    fn or_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2014        crate::kernel!(
2015            #[inline(always)]
2016            fn kernel(token: Sse2, a: u8x16<Sse2>, b: u8x16<Sse2>) -> u8x16<Sse2> {
2017                _mm_or_si128(a.into(), b.into()).simd_into(token)
2018            }
2019        );
2020        kernel(self, a, b)
2021    }
2022    #[inline(always)]
2023    fn xor_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2024        crate::kernel!(
2025            #[inline(always)]
2026            fn kernel(token: Sse2, a: u8x16<Sse2>, b: u8x16<Sse2>) -> u8x16<Sse2> {
2027                _mm_xor_si128(a.into(), b.into()).simd_into(token)
2028            }
2029        );
2030        kernel(self, a, b)
2031    }
2032    #[inline(always)]
2033    fn not_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
2034        a ^ !0
2035    }
2036    #[inline(always)]
2037    fn shl_u8x16(self, a: u8x16<Self>, shift: u32) -> u8x16<Self> {
2038        crate::kernel!(
2039            #[inline(always)]
2040            fn kernel(token: Sse2, a: u8x16<Sse2>, shift: u32) -> u8x16<Sse2> {
2041                let val = a.into();
2042                let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
2043                let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
2044                let byte_mask = _mm_set1_epi8(mask_byte);
2045                _mm_sll_epi16(_mm_and_si128(val, byte_mask), shift_count).simd_into(token)
2046            }
2047        );
2048        kernel(self, a, shift)
2049    }
2050    #[inline(always)]
2051    fn shlv_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2052        [
2053            u8::wrapping_shl(a[0usize], b[0usize] as u32),
2054            u8::wrapping_shl(a[1usize], b[1usize] as u32),
2055            u8::wrapping_shl(a[2usize], b[2usize] as u32),
2056            u8::wrapping_shl(a[3usize], b[3usize] as u32),
2057            u8::wrapping_shl(a[4usize], b[4usize] as u32),
2058            u8::wrapping_shl(a[5usize], b[5usize] as u32),
2059            u8::wrapping_shl(a[6usize], b[6usize] as u32),
2060            u8::wrapping_shl(a[7usize], b[7usize] as u32),
2061            u8::wrapping_shl(a[8usize], b[8usize] as u32),
2062            u8::wrapping_shl(a[9usize], b[9usize] as u32),
2063            u8::wrapping_shl(a[10usize], b[10usize] as u32),
2064            u8::wrapping_shl(a[11usize], b[11usize] as u32),
2065            u8::wrapping_shl(a[12usize], b[12usize] as u32),
2066            u8::wrapping_shl(a[13usize], b[13usize] as u32),
2067            u8::wrapping_shl(a[14usize], b[14usize] as u32),
2068            u8::wrapping_shl(a[15usize], b[15usize] as u32),
2069        ]
2070        .simd_into(self)
2071    }
2072    #[inline(always)]
2073    fn shr_u8x16(self, a: u8x16<Self>, shift: u32) -> u8x16<Self> {
2074        crate::kernel!(
2075            #[inline(always)]
2076            fn kernel(token: Sse2, a: u8x16<Sse2>, shift: u32) -> u8x16<Sse2> {
2077                let val = a.into();
2078                let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
2079                let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
2080                let byte_mask = _mm_set1_epi8(mask_byte);
2081                let shifted = _mm_srl_epi16(val, shift_count);
2082                let result = { _mm_and_si128(shifted, byte_mask) };
2083                result.simd_into(token)
2084            }
2085        );
2086        kernel(self, a, shift)
2087    }
2088    #[inline(always)]
2089    fn shrv_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2090        [
2091            u8::wrapping_shr(a[0usize], b[0usize] as u32),
2092            u8::wrapping_shr(a[1usize], b[1usize] as u32),
2093            u8::wrapping_shr(a[2usize], b[2usize] as u32),
2094            u8::wrapping_shr(a[3usize], b[3usize] as u32),
2095            u8::wrapping_shr(a[4usize], b[4usize] as u32),
2096            u8::wrapping_shr(a[5usize], b[5usize] as u32),
2097            u8::wrapping_shr(a[6usize], b[6usize] as u32),
2098            u8::wrapping_shr(a[7usize], b[7usize] as u32),
2099            u8::wrapping_shr(a[8usize], b[8usize] as u32),
2100            u8::wrapping_shr(a[9usize], b[9usize] as u32),
2101            u8::wrapping_shr(a[10usize], b[10usize] as u32),
2102            u8::wrapping_shr(a[11usize], b[11usize] as u32),
2103            u8::wrapping_shr(a[12usize], b[12usize] as u32),
2104            u8::wrapping_shr(a[13usize], b[13usize] as u32),
2105            u8::wrapping_shr(a[14usize], b[14usize] as u32),
2106            u8::wrapping_shr(a[15usize], b[15usize] as u32),
2107        ]
2108        .simd_into(self)
2109    }
2110    #[inline(always)]
2111    fn reduce_max_u8x16(self, a: u8x16<Self>) -> u8 {
2112        crate::kernel!(
2113            #[inline(always)]
2114            fn kernel(token: Sse2, a: u8x16<Sse2>) -> u8 {
2115                let reduced: __m128i = a.into();
2116                let shifted = _mm_srli_si128::<8>(reduced);
2117                let reduced = { _mm_max_epu8(reduced, shifted) };
2118                let shifted = _mm_srli_si128::<4>(reduced);
2119                let reduced = { _mm_max_epu8(reduced, shifted) };
2120                let shifted = _mm_srli_si128::<2>(reduced);
2121                let reduced = { _mm_max_epu8(reduced, shifted) };
2122                let shifted = _mm_srli_si128::<1>(reduced);
2123                let reduced = { _mm_max_epu8(reduced, shifted) };
2124                {
2125                    let lanes: [u8; 16usize] = crate::transmute::checked_transmute_copy(&reduced);
2126                    lanes[0]
2127                }
2128            }
2129        );
2130        kernel(self, a)
2131    }
2132    #[inline(always)]
2133    fn reduce_min_u8x16(self, a: u8x16<Self>) -> u8 {
2134        crate::kernel!(
2135            #[inline(always)]
2136            fn kernel(token: Sse2, a: u8x16<Sse2>) -> u8 {
2137                let reduced: __m128i = a.into();
2138                let shifted = _mm_srli_si128::<8>(reduced);
2139                let reduced = { _mm_min_epu8(reduced, shifted) };
2140                let shifted = _mm_srli_si128::<4>(reduced);
2141                let reduced = { _mm_min_epu8(reduced, shifted) };
2142                let shifted = _mm_srli_si128::<2>(reduced);
2143                let reduced = { _mm_min_epu8(reduced, shifted) };
2144                let shifted = _mm_srli_si128::<1>(reduced);
2145                let reduced = { _mm_min_epu8(reduced, shifted) };
2146                {
2147                    let lanes: [u8; 16usize] = crate::transmute::checked_transmute_copy(&reduced);
2148                    lanes[0]
2149                }
2150            }
2151        );
2152        kernel(self, a)
2153    }
2154    #[inline(always)]
2155    fn reduce_sum_u8x16(self, a: u8x16<Self>) -> u8 {
2156        crate::kernel!(
2157            #[inline(always)]
2158            fn kernel(token: Sse2, a: u8x16<Sse2>) -> u8 {
2159                let sum: __m128i = a.into();
2160                let sum = _mm_add_epi8(sum, _mm_srli_si128::<8>(sum));
2161                let sum = _mm_add_epi8(sum, _mm_srli_si128::<4>(sum));
2162                let sum = _mm_add_epi8(sum, _mm_srli_si128::<2>(sum));
2163                let sum = _mm_add_epi8(sum, _mm_srli_si128::<1>(sum));
2164                let lanes: [u8; 16usize] = crate::transmute::checked_transmute_copy(&sum);
2165                lanes[0]
2166            }
2167        );
2168        kernel(self, a)
2169    }
2170    #[inline(always)]
2171    fn reduce_product_u8x16(self, a: u8x16<Self>) -> u8 {
2172        crate::kernel!(
2173            #[inline(always)]
2174            fn kernel(token: Sse2, a: u8x16<Sse2>) -> u8 {
2175                let value: __m128i = a.into();
2176                let product = _mm_mullo_epi16(
2177                    _mm_unpacklo_epi8(value, value),
2178                    _mm_unpackhi_epi8(value, value),
2179                );
2180                let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b11_10_11_10>(product));
2181                let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b01_01_01_01>(product));
2182                let product = _mm_mullo_epi16(product, _mm_srli_epi32::<16>(product));
2183                let lanes: [u8; 16usize] = crate::transmute::checked_transmute_copy(&product);
2184                lanes[0]
2185            }
2186        );
2187        kernel(self, a)
2188    }
2189    #[inline(always)]
2190    fn max_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2191        crate::kernel!(
2192            #[inline(always)]
2193            fn kernel(token: Sse2, a: u8x16<Sse2>, b: u8x16<Sse2>) -> u8x16<Sse2> {
2194                _mm_max_epu8(a.into(), b.into()).simd_into(token)
2195            }
2196        );
2197        kernel(self, a, b)
2198    }
2199    #[inline(always)]
2200    fn min_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2201        crate::kernel!(
2202            #[inline(always)]
2203            fn kernel(token: Sse2, a: u8x16<Sse2>, b: u8x16<Sse2>) -> u8x16<Sse2> {
2204                _mm_min_epu8(a.into(), b.into()).simd_into(token)
2205            }
2206        );
2207        kernel(self, a, b)
2208    }
2209    #[inline(always)]
2210    fn clamp_u8x16(self, a: u8x16<Self>, b: u8x16<Self>, c: u8x16<Self>) -> u8x16<Self> {
2211        debug_assert!(
2212            b.simd_le(c).all_true(),
2213            "clamp requires min <= max for all elements"
2214        );
2215        a.max(b).min(c)
2216    }
2217    #[inline(always)]
2218    fn simd_eq_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
2219        crate::kernel!(
2220            #[inline(always)]
2221            fn kernel(token: Sse2, a: u8x16<Sse2>, b: u8x16<Sse2>) -> mask8x16<Sse2> {
2222                _mm_cmpeq_epi8(a.into(), b.into()).simd_into(token)
2223            }
2224        );
2225        kernel(self, a, b)
2226    }
2227    #[inline(always)]
2228    fn simd_lt_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
2229        crate::kernel!(
2230            #[inline(always)]
2231            fn kernel(token: Sse2, a: u8x16<Sse2>, b: u8x16<Sse2>) -> mask8x16<Sse2> {
2232                {
2233                    let sign_bit = _mm_set1_epi8(0x80u8.cast_signed());
2234                    let lhs_signed = _mm_xor_si128(b.into(), sign_bit);
2235                    let rhs_signed = _mm_xor_si128(a.into(), sign_bit);
2236                    _mm_cmpgt_epi8(lhs_signed, rhs_signed)
2237                }
2238                .simd_into(token)
2239            }
2240        );
2241        kernel(self, a, b)
2242    }
2243    #[inline(always)]
2244    fn simd_le_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
2245        crate::kernel!(
2246            #[inline(always)]
2247            fn kernel(token: Sse2, a: u8x16<Sse2>, b: u8x16<Sse2>) -> mask8x16<Sse2> {
2248                {
2249                    let all_ones = _mm_set1_epi8(-1);
2250                    _mm_xor_si128(
2251                        {
2252                            let sign_bit = _mm_set1_epi8(0x80u8.cast_signed());
2253                            let lhs_signed = _mm_xor_si128(a.into(), sign_bit);
2254                            let rhs_signed = _mm_xor_si128(b.into(), sign_bit);
2255                            _mm_cmpgt_epi8(lhs_signed, rhs_signed)
2256                        },
2257                        all_ones,
2258                    )
2259                }
2260                .simd_into(token)
2261            }
2262        );
2263        kernel(self, a, b)
2264    }
2265    #[inline(always)]
2266    fn zip_low_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2267        crate::kernel!(
2268            #[inline(always)]
2269            fn kernel(token: Sse2, a: u8x16<Sse2>, b: u8x16<Sse2>) -> u8x16<Sse2> {
2270                _mm_unpacklo_epi8(a.into(), b.into()).simd_into(token)
2271            }
2272        );
2273        kernel(self, a, b)
2274    }
2275    #[inline(always)]
2276    fn zip_high_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2277        crate::kernel!(
2278            #[inline(always)]
2279            fn kernel(token: Sse2, a: u8x16<Sse2>, b: u8x16<Sse2>) -> u8x16<Sse2> {
2280                _mm_unpackhi_epi8(a.into(), b.into()).simd_into(token)
2281            }
2282        );
2283        kernel(self, a, b)
2284    }
2285    #[inline(always)]
2286    fn unzip_low_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2287        [
2288            a[0usize], a[2usize], a[4usize], a[6usize], a[8usize], a[10usize], a[12usize],
2289            a[14usize], b[0usize], b[2usize], b[4usize], b[6usize], b[8usize], b[10usize],
2290            b[12usize], b[14usize],
2291        ]
2292        .simd_into(self)
2293    }
2294    #[inline(always)]
2295    fn unzip_high_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
2296        [
2297            a[1usize], a[3usize], a[5usize], a[7usize], a[9usize], a[11usize], a[13usize],
2298            a[15usize], b[1usize], b[3usize], b[5usize], b[7usize], b[9usize], b[11usize],
2299            b[13usize], b[15usize],
2300        ]
2301        .simd_into(self)
2302    }
2303    #[inline(always)]
2304    fn interleave_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u8x16<Self>, u8x16<Self>) {
2305        (self.zip_low_u8x16(a, b), self.zip_high_u8x16(a, b))
2306    }
2307    #[inline(always)]
2308    fn deinterleave_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u8x16<Self>, u8x16<Self>) {
2309        (self.unzip_low_u8x16(a, b), self.unzip_high_u8x16(a, b))
2310    }
2311    #[inline(always)]
2312    fn select_u8x16(self, a: mask8x16<Self>, b: u8x16<Self>, c: u8x16<Self>) -> u8x16<Self> {
2313        crate::kernel!(
2314            #[inline(always)]
2315            fn kernel(
2316                token: Sse2,
2317                a: mask8x16<Sse2>,
2318                b: u8x16<Sse2>,
2319                c: u8x16<Sse2>,
2320            ) -> u8x16<Sse2> {
2321                _mm_or_si128(
2322                    _mm_and_si128(a.into(), b.into()),
2323                    _mm_andnot_si128(a.into(), c.into()),
2324                )
2325                .simd_into(token)
2326            }
2327        );
2328        kernel(self, a, b, c)
2329    }
2330    #[inline(always)]
2331    fn combine_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x32<Self> {
2332        u8x32 {
2333            val: crate::support::Aligned256([a.val.0, b.val.0]),
2334            simd: self,
2335        }
2336    }
2337    #[inline(always)]
2338    fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16<Self>; 4usize] {
2339        [
2340            [
2341                src[0usize],
2342                src[4usize],
2343                src[8usize],
2344                src[12usize],
2345                src[16usize],
2346                src[20usize],
2347                src[24usize],
2348                src[28usize],
2349                src[32usize],
2350                src[36usize],
2351                src[40usize],
2352                src[44usize],
2353                src[48usize],
2354                src[52usize],
2355                src[56usize],
2356                src[60usize],
2357            ]
2358            .simd_into(self),
2359            [
2360                src[1usize],
2361                src[5usize],
2362                src[9usize],
2363                src[13usize],
2364                src[17usize],
2365                src[21usize],
2366                src[25usize],
2367                src[29usize],
2368                src[33usize],
2369                src[37usize],
2370                src[41usize],
2371                src[45usize],
2372                src[49usize],
2373                src[53usize],
2374                src[57usize],
2375                src[61usize],
2376            ]
2377            .simd_into(self),
2378            [
2379                src[2usize],
2380                src[6usize],
2381                src[10usize],
2382                src[14usize],
2383                src[18usize],
2384                src[22usize],
2385                src[26usize],
2386                src[30usize],
2387                src[34usize],
2388                src[38usize],
2389                src[42usize],
2390                src[46usize],
2391                src[50usize],
2392                src[54usize],
2393                src[58usize],
2394                src[62usize],
2395            ]
2396            .simd_into(self),
2397            [
2398                src[3usize],
2399                src[7usize],
2400                src[11usize],
2401                src[15usize],
2402                src[19usize],
2403                src[23usize],
2404                src[27usize],
2405                src[31usize],
2406                src[35usize],
2407                src[39usize],
2408                src[43usize],
2409                src[47usize],
2410                src[51usize],
2411                src[55usize],
2412                src[59usize],
2413                src[63usize],
2414            ]
2415            .simd_into(self),
2416        ]
2417    }
2418    #[inline(always)]
2419    fn store_four_interleaved_u8x16(
2420        self,
2421        vectors: [u8x16<Self>; 4usize],
2422        dest: &mut [u8; 64usize],
2423    ) -> () {
2424        *dest = [
2425            vectors[0usize][0usize],
2426            vectors[1usize][0usize],
2427            vectors[2usize][0usize],
2428            vectors[3usize][0usize],
2429            vectors[0usize][1usize],
2430            vectors[1usize][1usize],
2431            vectors[2usize][1usize],
2432            vectors[3usize][1usize],
2433            vectors[0usize][2usize],
2434            vectors[1usize][2usize],
2435            vectors[2usize][2usize],
2436            vectors[3usize][2usize],
2437            vectors[0usize][3usize],
2438            vectors[1usize][3usize],
2439            vectors[2usize][3usize],
2440            vectors[3usize][3usize],
2441            vectors[0usize][4usize],
2442            vectors[1usize][4usize],
2443            vectors[2usize][4usize],
2444            vectors[3usize][4usize],
2445            vectors[0usize][5usize],
2446            vectors[1usize][5usize],
2447            vectors[2usize][5usize],
2448            vectors[3usize][5usize],
2449            vectors[0usize][6usize],
2450            vectors[1usize][6usize],
2451            vectors[2usize][6usize],
2452            vectors[3usize][6usize],
2453            vectors[0usize][7usize],
2454            vectors[1usize][7usize],
2455            vectors[2usize][7usize],
2456            vectors[3usize][7usize],
2457            vectors[0usize][8usize],
2458            vectors[1usize][8usize],
2459            vectors[2usize][8usize],
2460            vectors[3usize][8usize],
2461            vectors[0usize][9usize],
2462            vectors[1usize][9usize],
2463            vectors[2usize][9usize],
2464            vectors[3usize][9usize],
2465            vectors[0usize][10usize],
2466            vectors[1usize][10usize],
2467            vectors[2usize][10usize],
2468            vectors[3usize][10usize],
2469            vectors[0usize][11usize],
2470            vectors[1usize][11usize],
2471            vectors[2usize][11usize],
2472            vectors[3usize][11usize],
2473            vectors[0usize][12usize],
2474            vectors[1usize][12usize],
2475            vectors[2usize][12usize],
2476            vectors[3usize][12usize],
2477            vectors[0usize][13usize],
2478            vectors[1usize][13usize],
2479            vectors[2usize][13usize],
2480            vectors[3usize][13usize],
2481            vectors[0usize][14usize],
2482            vectors[1usize][14usize],
2483            vectors[2usize][14usize],
2484            vectors[3usize][14usize],
2485            vectors[0usize][15usize],
2486            vectors[1usize][15usize],
2487            vectors[2usize][15usize],
2488            vectors[3usize][15usize],
2489        ];
2490    }
2491    #[inline(always)]
2492    fn widen_u8x16(self, a: u8x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
2493        crate::kernel!(
2494            #[inline(always)]
2495            fn kernel(token: Sse2, a: u8x16<Sse2>) -> (u16x8<Sse2>, u16x8<Sse2>) {
2496                let raw = a.into();
2497                let sign = _mm_setzero_si128();
2498                (
2499                    _mm_unpacklo_epi8(raw, sign).simd_into(token),
2500                    _mm_unpackhi_epi8(raw, sign).simd_into(token),
2501                )
2502            }
2503        );
2504        kernel(self, a)
2505    }
2506    #[inline(always)]
2507    fn widening_mul_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
2508        let (a0, a1) = self.widen_u8x16(a);
2509        let (b0, b1) = self.widen_u8x16(b);
2510        (self.mul_u16x8(a0, b0), self.mul_u16x8(a1, b1))
2511    }
2512    #[inline(always)]
2513    fn widening_add_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
2514        let (a0, a1) = self.widen_u8x16(a);
2515        let (b0, b1) = self.widen_u8x16(b);
2516        (self.add_u16x8(a0, b0), self.add_u16x8(a1, b1))
2517    }
2518    #[inline(always)]
2519    fn pairwise_widening_add_u8x16(self, a: u8x16<Self>) -> u16x8<Self> {
2520        [
2521            (a[0usize] as u16) + (a[1usize] as u16),
2522            (a[2usize] as u16) + (a[3usize] as u16),
2523            (a[4usize] as u16) + (a[5usize] as u16),
2524            (a[6usize] as u16) + (a[7usize] as u16),
2525            (a[8usize] as u16) + (a[9usize] as u16),
2526            (a[10usize] as u16) + (a[11usize] as u16),
2527            (a[12usize] as u16) + (a[13usize] as u16),
2528            (a[14usize] as u16) + (a[15usize] as u16),
2529        ]
2530        .simd_into(self)
2531    }
2532    #[inline(always)]
2533    fn pairwise_dot_product_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u16x8<Self> {
2534        let (low, high) = self.widening_mul_u8x16(a, b);
2535        let (even, odd) = self.deinterleave_u16x8(low, high);
2536        self.add_u16x8(even, odd)
2537    }
2538    #[inline(always)]
2539    fn splat_mask8x16(self, val: bool) -> mask8x16<Self> {
2540        crate::kernel!(
2541            #[inline(always)]
2542            fn kernel(token: Sse2, val: bool) -> mask8x16<Sse2> {
2543                let val: i8 = if val { !0 } else { 0 };
2544                _mm_set1_epi8(val).simd_into(token)
2545            }
2546        );
2547        kernel(self, val)
2548    }
2549    #[inline(always)]
2550    fn from_bitmask_mask8x16(self, bits: u64) -> mask8x16<Self> {
2551        let lanes: [i8; 16usize] = [
2552            if bits & 1 != 0 { !0 } else { 0 },
2553            if (bits >> 1usize) & 1 != 0 { !0 } else { 0 },
2554            if (bits >> 2usize) & 1 != 0 { !0 } else { 0 },
2555            if (bits >> 3usize) & 1 != 0 { !0 } else { 0 },
2556            if (bits >> 4usize) & 1 != 0 { !0 } else { 0 },
2557            if (bits >> 5usize) & 1 != 0 { !0 } else { 0 },
2558            if (bits >> 6usize) & 1 != 0 { !0 } else { 0 },
2559            if (bits >> 7usize) & 1 != 0 { !0 } else { 0 },
2560            if (bits >> 8usize) & 1 != 0 { !0 } else { 0 },
2561            if (bits >> 9usize) & 1 != 0 { !0 } else { 0 },
2562            if (bits >> 10usize) & 1 != 0 { !0 } else { 0 },
2563            if (bits >> 11usize) & 1 != 0 { !0 } else { 0 },
2564            if (bits >> 12usize) & 1 != 0 { !0 } else { 0 },
2565            if (bits >> 13usize) & 1 != 0 { !0 } else { 0 },
2566            if (bits >> 14usize) & 1 != 0 { !0 } else { 0 },
2567            if (bits >> 15usize) & 1 != 0 { !0 } else { 0 },
2568        ];
2569        lanes.simd_into(self)
2570    }
2571    #[inline(always)]
2572    fn to_bitmask_mask8x16(self, a: mask8x16<Self>) -> u64 {
2573        crate::kernel!(
2574            #[inline(always)]
2575            fn kernel(token: Sse2, a: mask8x16<Sse2>) -> u64 {
2576                _mm_movemask_epi8(a.into()) as u32 as u64
2577            }
2578        );
2579        kernel(self, a)
2580    }
2581    #[inline(always)]
2582    fn from_vector_mask8x16(self, vector: i8x16<Self>) -> mask8x16<Self> {
2583        mask8x16 {
2584            val: vector.val,
2585            simd: self,
2586        }
2587    }
2588    #[inline(always)]
2589    fn to_vector_mask8x16(self, a: mask8x16<Self>) -> i8x16<Self> {
2590        i8x16 {
2591            val: a.val,
2592            simd: self,
2593        }
2594    }
2595    #[inline(always)]
2596    fn set_mask8x16(self, a: &mut mask8x16<Self>, index: usize, value: bool) -> () {
2597        assert!(
2598            index < 16usize,
2599            "mask lane index {index} is out of bounds for {} lanes",
2600            16usize
2601        );
2602        let mut lanes: [i8; 16usize] = (*a).into();
2603        lanes[index] = if value { !0 } else { 0 };
2604        *a = lanes.simd_into(self);
2605    }
2606    #[inline(always)]
2607    fn rotate_elements_left_mask8x16<const OFFSET: usize>(
2608        self,
2609        a: mask8x16<Self>,
2610    ) -> mask8x16<Self> {
2611        let int = i8x16 {
2612            val: crate::transmute::checked_transmute_copy(&a.val),
2613            simd: self,
2614        };
2615        let rotated = int.rotate_elements_left::<OFFSET>();
2616        mask8x16 {
2617            val: crate::transmute::checked_transmute_copy(&rotated.val),
2618            simd: self,
2619        }
2620    }
2621    #[inline(always)]
2622    fn rotate_elements_right_mask8x16<const OFFSET: usize>(
2623        self,
2624        a: mask8x16<Self>,
2625    ) -> mask8x16<Self> {
2626        let int = i8x16 {
2627            val: crate::transmute::checked_transmute_copy(&a.val),
2628            simd: self,
2629        };
2630        let rotated = int.rotate_elements_right::<OFFSET>();
2631        mask8x16 {
2632            val: crate::transmute::checked_transmute_copy(&rotated.val),
2633            simd: self,
2634        }
2635    }
2636    #[inline(always)]
2637    fn and_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
2638        crate::kernel!(
2639            #[inline(always)]
2640            fn kernel(token: Sse2, a: mask8x16<Sse2>, b: mask8x16<Sse2>) -> mask8x16<Sse2> {
2641                _mm_and_si128(a.into(), b.into()).simd_into(token)
2642            }
2643        );
2644        kernel(self, a, b)
2645    }
2646    #[inline(always)]
2647    fn or_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
2648        crate::kernel!(
2649            #[inline(always)]
2650            fn kernel(token: Sse2, a: mask8x16<Sse2>, b: mask8x16<Sse2>) -> mask8x16<Sse2> {
2651                _mm_or_si128(a.into(), b.into()).simd_into(token)
2652            }
2653        );
2654        kernel(self, a, b)
2655    }
2656    #[inline(always)]
2657    fn xor_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
2658        crate::kernel!(
2659            #[inline(always)]
2660            fn kernel(token: Sse2, a: mask8x16<Sse2>, b: mask8x16<Sse2>) -> mask8x16<Sse2> {
2661                _mm_xor_si128(a.into(), b.into()).simd_into(token)
2662            }
2663        );
2664        kernel(self, a, b)
2665    }
2666    #[inline(always)]
2667    fn not_mask8x16(self, a: mask8x16<Self>) -> mask8x16<Self> {
2668        self.xor_mask8x16(a, self.splat_mask8x16(true))
2669    }
2670    #[inline(always)]
2671    fn reverse_mask8x16(self, a: mask8x16<Self>) -> mask8x16<Self> {
2672        let lanes = i8x16 {
2673            val: crate::transmute::checked_transmute_copy(&a.val),
2674            simd: self,
2675        };
2676        let reversed = self.reverse_i8x16(lanes);
2677        mask8x16 {
2678            val: crate::transmute::checked_transmute_copy(&reversed.val),
2679            simd: self,
2680        }
2681    }
2682    #[inline(always)]
2683    fn select_mask8x16(
2684        self,
2685        a: mask8x16<Self>,
2686        b: mask8x16<Self>,
2687        c: mask8x16<Self>,
2688    ) -> mask8x16<Self> {
2689        crate::kernel!(
2690            #[inline(always)]
2691            fn kernel(
2692                token: Sse2,
2693                a: mask8x16<Sse2>,
2694                b: mask8x16<Sse2>,
2695                c: mask8x16<Sse2>,
2696            ) -> mask8x16<Sse2> {
2697                _mm_or_si128(
2698                    _mm_and_si128(a.into(), b.into()),
2699                    _mm_andnot_si128(a.into(), c.into()),
2700                )
2701                .simd_into(token)
2702            }
2703        );
2704        kernel(self, a, b, c)
2705    }
2706    #[inline(always)]
2707    fn simd_eq_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
2708        crate::kernel!(
2709            #[inline(always)]
2710            fn kernel(token: Sse2, a: mask8x16<Sse2>, b: mask8x16<Sse2>) -> mask8x16<Sse2> {
2711                _mm_cmpeq_epi8(a.into(), b.into()).simd_into(token)
2712            }
2713        );
2714        kernel(self, a, b)
2715    }
2716    #[inline(always)]
2717    fn any_true_mask8x16(self, a: mask8x16<Self>) -> bool {
2718        crate::kernel!(
2719            #[inline(always)]
2720            fn kernel(token: Sse2, a: mask8x16<Sse2>) -> bool {
2721                _mm_movemask_epi8(a.into()) as u32 != 0
2722            }
2723        );
2724        kernel(self, a)
2725    }
2726    #[inline(always)]
2727    fn all_true_mask8x16(self, a: mask8x16<Self>) -> bool {
2728        crate::kernel!(
2729            #[inline(always)]
2730            fn kernel(token: Sse2, a: mask8x16<Sse2>) -> bool {
2731                _mm_movemask_epi8(a.into()) as u32 == 0xffff
2732            }
2733        );
2734        kernel(self, a)
2735    }
2736    #[inline(always)]
2737    fn any_false_mask8x16(self, a: mask8x16<Self>) -> bool {
2738        crate::kernel!(
2739            #[inline(always)]
2740            fn kernel(token: Sse2, a: mask8x16<Sse2>) -> bool {
2741                _mm_movemask_epi8(a.into()) as u32 != 0xffff
2742            }
2743        );
2744        kernel(self, a)
2745    }
2746    #[inline(always)]
2747    fn all_false_mask8x16(self, a: mask8x16<Self>) -> bool {
2748        crate::kernel!(
2749            #[inline(always)]
2750            fn kernel(token: Sse2, a: mask8x16<Sse2>) -> bool {
2751                _mm_movemask_epi8(a.into()) as u32 == 0
2752            }
2753        );
2754        kernel(self, a)
2755    }
2756    #[inline(always)]
2757    fn combine_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x32<Self> {
2758        mask8x32 {
2759            val: crate::support::Aligned256([a.val.0, b.val.0]),
2760            simd: self,
2761        }
2762    }
2763    #[inline(always)]
2764    fn widen_mask8x16(self, a: mask8x16<Self>) -> (mask16x8<Self>, mask16x8<Self>) {
2765        crate::kernel!(
2766            #[inline(always)]
2767            fn kernel(token: Sse2, a: mask8x16<Sse2>) -> (mask16x8<Sse2>, mask16x8<Sse2>) {
2768                let raw = a.into();
2769                (
2770                    _mm_unpacklo_epi8(raw, raw).simd_into(token),
2771                    _mm_unpackhi_epi8(raw, raw).simd_into(token),
2772                )
2773            }
2774        );
2775        kernel(self, a)
2776    }
2777    #[inline(always)]
2778    fn abs_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2779        [
2780            i16::wrapping_abs(a[0usize]),
2781            i16::wrapping_abs(a[1usize]),
2782            i16::wrapping_abs(a[2usize]),
2783            i16::wrapping_abs(a[3usize]),
2784            i16::wrapping_abs(a[4usize]),
2785            i16::wrapping_abs(a[5usize]),
2786            i16::wrapping_abs(a[6usize]),
2787            i16::wrapping_abs(a[7usize]),
2788        ]
2789        .simd_into(self)
2790    }
2791    #[inline(always)]
2792    fn splat_i16x8(self, val: i16) -> i16x8<Self> {
2793        crate::kernel!(
2794            #[inline(always)]
2795            fn kernel(token: Sse2, val: i16) -> i16x8<Sse2> {
2796                _mm_set1_epi16(val).simd_into(token)
2797            }
2798        );
2799        kernel(self, val)
2800    }
2801    #[inline(always)]
2802    fn reverse_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2803        [
2804            a[7usize], a[6usize], a[5usize], a[4usize], a[3usize], a[2usize], a[1usize], a[0usize],
2805        ]
2806        .simd_into(self)
2807    }
2808    #[inline(always)]
2809    fn slide_i16x8<const SHIFT: usize>(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2810        if SHIFT >= 8usize {
2811            return b;
2812        }
2813        let result = dyn_alignr_128(
2814            self,
2815            Bytes::to_bytes(b).val.0,
2816            Bytes::to_bytes(a).val.0,
2817            SHIFT * 2usize,
2818        );
2819        Bytes::from_bytes(u8x16 {
2820            val: crate::support::Aligned128(result),
2821            simd: self,
2822        })
2823    }
2824    #[inline(always)]
2825    fn compress_i16x8(self, values: i16x8<Self>, mask: mask16x8<Self>) -> i16x8<Self> {
2826        let mask = mask8x16 {
2827            val: crate::transmute::checked_transmute_copy(&mask.val),
2828            simd: self,
2829        };
2830        Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask))
2831    }
2832    #[inline(always)]
2833    fn compress_merge_i16x8(
2834        self,
2835        values: i16x8<Self>,
2836        mask: mask16x8<Self>,
2837        merge: i16x8<Self>,
2838    ) -> i16x8<Self> {
2839        let mask = mask8x16 {
2840            val: crate::transmute::checked_transmute_copy(&mask.val),
2841            simd: self,
2842        };
2843        Bytes::from_bytes(self.compress_merge_u8x16(
2844            Bytes::to_bytes(values),
2845            mask,
2846            Bytes::to_bytes(merge),
2847        ))
2848    }
2849    #[inline(always)]
2850    fn expand_i16x8(self, values: i16x8<Self>, mask: mask16x8<Self>) -> i16x8<Self> {
2851        let mask = mask8x16 {
2852            val: crate::transmute::checked_transmute_copy(&mask.val),
2853            simd: self,
2854        };
2855        Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask))
2856    }
2857    #[inline(always)]
2858    fn expand_merge_i16x8(
2859        self,
2860        values: i16x8<Self>,
2861        mask: mask16x8<Self>,
2862        merge: i16x8<Self>,
2863    ) -> i16x8<Self> {
2864        let mask = mask8x16 {
2865            val: crate::transmute::checked_transmute_copy(&mask.val),
2866            simd: self,
2867        };
2868        Bytes::from_bytes(self.expand_merge_u8x16(
2869            Bytes::to_bytes(values),
2870            mask,
2871            Bytes::to_bytes(merge),
2872        ))
2873    }
2874    #[inline(always)]
2875    fn count_ones_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2876        [
2877            i16::try_from(a[0usize].count_ones()).unwrap(),
2878            i16::try_from(a[1usize].count_ones()).unwrap(),
2879            i16::try_from(a[2usize].count_ones()).unwrap(),
2880            i16::try_from(a[3usize].count_ones()).unwrap(),
2881            i16::try_from(a[4usize].count_ones()).unwrap(),
2882            i16::try_from(a[5usize].count_ones()).unwrap(),
2883            i16::try_from(a[6usize].count_ones()).unwrap(),
2884            i16::try_from(a[7usize].count_ones()).unwrap(),
2885        ]
2886        .simd_into(self)
2887    }
2888    #[inline(always)]
2889    fn count_zeros_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2890        [
2891            i16::try_from(a[0usize].count_zeros()).unwrap(),
2892            i16::try_from(a[1usize].count_zeros()).unwrap(),
2893            i16::try_from(a[2usize].count_zeros()).unwrap(),
2894            i16::try_from(a[3usize].count_zeros()).unwrap(),
2895            i16::try_from(a[4usize].count_zeros()).unwrap(),
2896            i16::try_from(a[5usize].count_zeros()).unwrap(),
2897            i16::try_from(a[6usize].count_zeros()).unwrap(),
2898            i16::try_from(a[7usize].count_zeros()).unwrap(),
2899        ]
2900        .simd_into(self)
2901    }
2902    #[inline(always)]
2903    fn add_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2904        crate::kernel!(
2905            #[inline(always)]
2906            fn kernel(token: Sse2, a: i16x8<Sse2>, b: i16x8<Sse2>) -> i16x8<Sse2> {
2907                _mm_add_epi16(a.into(), b.into()).simd_into(token)
2908            }
2909        );
2910        kernel(self, a, b)
2911    }
2912    #[inline(always)]
2913    fn saturating_add_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2914        crate::kernel!(
2915            #[inline(always)]
2916            fn kernel(token: Sse2, a: i16x8<Sse2>, b: i16x8<Sse2>) -> i16x8<Sse2> {
2917                _mm_adds_epi16(a.into(), b.into()).simd_into(token)
2918            }
2919        );
2920        kernel(self, a, b)
2921    }
2922    #[inline(always)]
2923    fn sub_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2924        crate::kernel!(
2925            #[inline(always)]
2926            fn kernel(token: Sse2, a: i16x8<Sse2>, b: i16x8<Sse2>) -> i16x8<Sse2> {
2927                _mm_sub_epi16(a.into(), b.into()).simd_into(token)
2928            }
2929        );
2930        kernel(self, a, b)
2931    }
2932    #[inline(always)]
2933    fn saturating_sub_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2934        crate::kernel!(
2935            #[inline(always)]
2936            fn kernel(token: Sse2, a: i16x8<Sse2>, b: i16x8<Sse2>) -> i16x8<Sse2> {
2937                _mm_subs_epi16(a.into(), b.into()).simd_into(token)
2938            }
2939        );
2940        kernel(self, a, b)
2941    }
2942    #[inline(always)]
2943    fn mul_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2944        crate::kernel!(
2945            #[inline(always)]
2946            fn kernel(token: Sse2, a: i16x8<Sse2>, b: i16x8<Sse2>) -> i16x8<Sse2> {
2947                _mm_mullo_epi16(a.into(), b.into()).simd_into(token)
2948            }
2949        );
2950        kernel(self, a, b)
2951    }
2952    #[inline(always)]
2953    fn and_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2954        crate::kernel!(
2955            #[inline(always)]
2956            fn kernel(token: Sse2, a: i16x8<Sse2>, b: i16x8<Sse2>) -> i16x8<Sse2> {
2957                _mm_and_si128(a.into(), b.into()).simd_into(token)
2958            }
2959        );
2960        kernel(self, a, b)
2961    }
2962    #[inline(always)]
2963    fn or_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2964        crate::kernel!(
2965            #[inline(always)]
2966            fn kernel(token: Sse2, a: i16x8<Sse2>, b: i16x8<Sse2>) -> i16x8<Sse2> {
2967                _mm_or_si128(a.into(), b.into()).simd_into(token)
2968            }
2969        );
2970        kernel(self, a, b)
2971    }
2972    #[inline(always)]
2973    fn xor_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2974        crate::kernel!(
2975            #[inline(always)]
2976            fn kernel(token: Sse2, a: i16x8<Sse2>, b: i16x8<Sse2>) -> i16x8<Sse2> {
2977                _mm_xor_si128(a.into(), b.into()).simd_into(token)
2978            }
2979        );
2980        kernel(self, a, b)
2981    }
2982    #[inline(always)]
2983    fn not_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
2984        a ^ !0
2985    }
2986    #[inline(always)]
2987    fn shl_i16x8(self, a: i16x8<Self>, shift: u32) -> i16x8<Self> {
2988        crate::kernel!(
2989            #[inline(always)]
2990            fn kernel(token: Sse2, a: i16x8<Sse2>, shift: u32) -> i16x8<Sse2> {
2991                _mm_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
2992            }
2993        );
2994        kernel(self, a, shift)
2995    }
2996    #[inline(always)]
2997    fn shlv_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
2998        [
2999            i16::wrapping_shl(a[0usize], b[0usize] as u32),
3000            i16::wrapping_shl(a[1usize], b[1usize] as u32),
3001            i16::wrapping_shl(a[2usize], b[2usize] as u32),
3002            i16::wrapping_shl(a[3usize], b[3usize] as u32),
3003            i16::wrapping_shl(a[4usize], b[4usize] as u32),
3004            i16::wrapping_shl(a[5usize], b[5usize] as u32),
3005            i16::wrapping_shl(a[6usize], b[6usize] as u32),
3006            i16::wrapping_shl(a[7usize], b[7usize] as u32),
3007        ]
3008        .simd_into(self)
3009    }
3010    #[inline(always)]
3011    fn shr_i16x8(self, a: i16x8<Self>, shift: u32) -> i16x8<Self> {
3012        crate::kernel!(
3013            #[inline(always)]
3014            fn kernel(token: Sse2, a: i16x8<Sse2>, shift: u32) -> i16x8<Sse2> {
3015                _mm_sra_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
3016            }
3017        );
3018        kernel(self, a, shift)
3019    }
3020    #[inline(always)]
3021    fn shrv_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3022        [
3023            i16::wrapping_shr(a[0usize], b[0usize] as u32),
3024            i16::wrapping_shr(a[1usize], b[1usize] as u32),
3025            i16::wrapping_shr(a[2usize], b[2usize] as u32),
3026            i16::wrapping_shr(a[3usize], b[3usize] as u32),
3027            i16::wrapping_shr(a[4usize], b[4usize] as u32),
3028            i16::wrapping_shr(a[5usize], b[5usize] as u32),
3029            i16::wrapping_shr(a[6usize], b[6usize] as u32),
3030            i16::wrapping_shr(a[7usize], b[7usize] as u32),
3031        ]
3032        .simd_into(self)
3033    }
3034    #[inline(always)]
3035    fn reduce_max_i16x8(self, a: i16x8<Self>) -> i16 {
3036        crate::kernel!(
3037            #[inline(always)]
3038            fn kernel(token: Sse2, a: i16x8<Sse2>) -> i16 {
3039                let reduced: __m128i = a.into();
3040                let shifted = _mm_srli_si128::<8>(reduced);
3041                let reduced = { _mm_max_epi16(reduced, shifted) };
3042                let shifted = _mm_srli_si128::<4>(reduced);
3043                let reduced = { _mm_max_epi16(reduced, shifted) };
3044                let shifted = _mm_srli_si128::<2>(reduced);
3045                let reduced = { _mm_max_epi16(reduced, shifted) };
3046                {
3047                    let lanes: [i16; 8usize] = crate::transmute::checked_transmute_copy(&reduced);
3048                    lanes[0]
3049                }
3050            }
3051        );
3052        kernel(self, a)
3053    }
3054    #[inline(always)]
3055    fn reduce_min_i16x8(self, a: i16x8<Self>) -> i16 {
3056        crate::kernel!(
3057            #[inline(always)]
3058            fn kernel(token: Sse2, a: i16x8<Sse2>) -> i16 {
3059                let reduced: __m128i = a.into();
3060                let shifted = _mm_srli_si128::<8>(reduced);
3061                let reduced = { _mm_min_epi16(reduced, shifted) };
3062                let shifted = _mm_srli_si128::<4>(reduced);
3063                let reduced = { _mm_min_epi16(reduced, shifted) };
3064                let shifted = _mm_srli_si128::<2>(reduced);
3065                let reduced = { _mm_min_epi16(reduced, shifted) };
3066                {
3067                    let lanes: [i16; 8usize] = crate::transmute::checked_transmute_copy(&reduced);
3068                    lanes[0]
3069                }
3070            }
3071        );
3072        kernel(self, a)
3073    }
3074    #[inline(always)]
3075    fn reduce_sum_i16x8(self, a: i16x8<Self>) -> i16 {
3076        crate::kernel!(
3077            #[inline(always)]
3078            fn kernel(token: Sse2, a: i16x8<Sse2>) -> i16 {
3079                let sum: __m128i = a.into();
3080                let sum = _mm_add_epi16(sum, _mm_srli_si128::<8>(sum));
3081                let sum = _mm_add_epi16(sum, _mm_srli_si128::<4>(sum));
3082                let sum = _mm_add_epi16(sum, _mm_srli_si128::<2>(sum));
3083                let lanes: [i16; 8usize] = crate::transmute::checked_transmute_copy(&sum);
3084                lanes[0]
3085            }
3086        );
3087        kernel(self, a)
3088    }
3089    #[inline(always)]
3090    fn reduce_product_i16x8(self, a: i16x8<Self>) -> i16 {
3091        crate::kernel!(
3092            #[inline(always)]
3093            fn kernel(token: Sse2, a: i16x8<Sse2>) -> i16 {
3094                let product: __m128i = a.into();
3095                let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b11_10_11_10>(product));
3096                let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b01_01_01_01>(product));
3097                let product = _mm_mullo_epi16(product, _mm_srli_epi32::<16>(product));
3098                let lanes: [i16; 8usize] = crate::transmute::checked_transmute_copy(&product);
3099                lanes[0]
3100            }
3101        );
3102        kernel(self, a)
3103    }
3104    #[inline(always)]
3105    fn max_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3106        crate::kernel!(
3107            #[inline(always)]
3108            fn kernel(token: Sse2, a: i16x8<Sse2>, b: i16x8<Sse2>) -> i16x8<Sse2> {
3109                _mm_max_epi16(a.into(), b.into()).simd_into(token)
3110            }
3111        );
3112        kernel(self, a, b)
3113    }
3114    #[inline(always)]
3115    fn min_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3116        crate::kernel!(
3117            #[inline(always)]
3118            fn kernel(token: Sse2, a: i16x8<Sse2>, b: i16x8<Sse2>) -> i16x8<Sse2> {
3119                _mm_min_epi16(a.into(), b.into()).simd_into(token)
3120            }
3121        );
3122        kernel(self, a, b)
3123    }
3124    #[inline(always)]
3125    fn clamp_i16x8(self, a: i16x8<Self>, b: i16x8<Self>, c: i16x8<Self>) -> i16x8<Self> {
3126        debug_assert!(
3127            b.simd_le(c).all_true(),
3128            "clamp requires min <= max for all elements"
3129        );
3130        a.max(b).min(c)
3131    }
3132    #[inline(always)]
3133    fn simd_eq_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
3134        crate::kernel!(
3135            #[inline(always)]
3136            fn kernel(token: Sse2, a: i16x8<Sse2>, b: i16x8<Sse2>) -> mask16x8<Sse2> {
3137                _mm_cmpeq_epi16(a.into(), b.into()).simd_into(token)
3138            }
3139        );
3140        kernel(self, a, b)
3141    }
3142    #[inline(always)]
3143    fn simd_lt_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
3144        crate::kernel!(
3145            #[inline(always)]
3146            fn kernel(token: Sse2, a: i16x8<Sse2>, b: i16x8<Sse2>) -> mask16x8<Sse2> {
3147                _mm_cmpgt_epi16(b.into(), a.into()).simd_into(token)
3148            }
3149        );
3150        kernel(self, a, b)
3151    }
3152    #[inline(always)]
3153    fn simd_le_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
3154        crate::kernel!(
3155            #[inline(always)]
3156            fn kernel(token: Sse2, a: i16x8<Sse2>, b: i16x8<Sse2>) -> mask16x8<Sse2> {
3157                {
3158                    let all_ones = _mm_set1_epi8(-1);
3159                    _mm_xor_si128(_mm_cmpgt_epi16(a.into(), b.into()), all_ones)
3160                }
3161                .simd_into(token)
3162            }
3163        );
3164        kernel(self, a, b)
3165    }
3166    #[inline(always)]
3167    fn zip_low_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3168        crate::kernel!(
3169            #[inline(always)]
3170            fn kernel(token: Sse2, a: i16x8<Sse2>, b: i16x8<Sse2>) -> i16x8<Sse2> {
3171                _mm_unpacklo_epi16(a.into(), b.into()).simd_into(token)
3172            }
3173        );
3174        kernel(self, a, b)
3175    }
3176    #[inline(always)]
3177    fn zip_high_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3178        crate::kernel!(
3179            #[inline(always)]
3180            fn kernel(token: Sse2, a: i16x8<Sse2>, b: i16x8<Sse2>) -> i16x8<Sse2> {
3181                _mm_unpackhi_epi16(a.into(), b.into()).simd_into(token)
3182            }
3183        );
3184        kernel(self, a, b)
3185    }
3186    #[inline(always)]
3187    fn unzip_low_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3188        [
3189            a[0usize], a[2usize], a[4usize], a[6usize], b[0usize], b[2usize], b[4usize], b[6usize],
3190        ]
3191        .simd_into(self)
3192    }
3193    #[inline(always)]
3194    fn unzip_high_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
3195        [
3196            a[1usize], a[3usize], a[5usize], a[7usize], b[1usize], b[3usize], b[5usize], b[7usize],
3197        ]
3198        .simd_into(self)
3199    }
3200    #[inline(always)]
3201    fn interleave_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i16x8<Self>, i16x8<Self>) {
3202        (self.zip_low_i16x8(a, b), self.zip_high_i16x8(a, b))
3203    }
3204    #[inline(always)]
3205    fn deinterleave_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i16x8<Self>, i16x8<Self>) {
3206        (self.unzip_low_i16x8(a, b), self.unzip_high_i16x8(a, b))
3207    }
3208    #[inline(always)]
3209    fn select_i16x8(self, a: mask16x8<Self>, b: i16x8<Self>, c: i16x8<Self>) -> i16x8<Self> {
3210        crate::kernel!(
3211            #[inline(always)]
3212            fn kernel(
3213                token: Sse2,
3214                a: mask16x8<Sse2>,
3215                b: i16x8<Sse2>,
3216                c: i16x8<Sse2>,
3217            ) -> i16x8<Sse2> {
3218                _mm_or_si128(
3219                    _mm_and_si128(a.into(), b.into()),
3220                    _mm_andnot_si128(a.into(), c.into()),
3221                )
3222                .simd_into(token)
3223            }
3224        );
3225        kernel(self, a, b, c)
3226    }
3227    #[inline(always)]
3228    fn combine_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x16<Self> {
3229        i16x16 {
3230            val: crate::support::Aligned256([a.val.0, b.val.0]),
3231            simd: self,
3232        }
3233    }
3234    #[inline(always)]
3235    fn neg_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
3236        crate::kernel!(
3237            #[inline(always)]
3238            fn kernel(token: Sse2, a: i16x8<Sse2>) -> i16x8<Sse2> {
3239                _mm_sub_epi16(_mm_setzero_si128(), a.into()).simd_into(token)
3240            }
3241        );
3242        kernel(self, a)
3243    }
3244    #[inline(always)]
3245    fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8<Self>; 4usize] {
3246        [
3247            [
3248                src[0usize],
3249                src[4usize],
3250                src[8usize],
3251                src[12usize],
3252                src[16usize],
3253                src[20usize],
3254                src[24usize],
3255                src[28usize],
3256            ]
3257            .simd_into(self),
3258            [
3259                src[1usize],
3260                src[5usize],
3261                src[9usize],
3262                src[13usize],
3263                src[17usize],
3264                src[21usize],
3265                src[25usize],
3266                src[29usize],
3267            ]
3268            .simd_into(self),
3269            [
3270                src[2usize],
3271                src[6usize],
3272                src[10usize],
3273                src[14usize],
3274                src[18usize],
3275                src[22usize],
3276                src[26usize],
3277                src[30usize],
3278            ]
3279            .simd_into(self),
3280            [
3281                src[3usize],
3282                src[7usize],
3283                src[11usize],
3284                src[15usize],
3285                src[19usize],
3286                src[23usize],
3287                src[27usize],
3288                src[31usize],
3289            ]
3290            .simd_into(self),
3291        ]
3292    }
3293    #[inline(always)]
3294    fn store_four_interleaved_i16x8(
3295        self,
3296        vectors: [i16x8<Self>; 4usize],
3297        dest: &mut [i16; 32usize],
3298    ) -> () {
3299        *dest = [
3300            vectors[0usize][0usize],
3301            vectors[1usize][0usize],
3302            vectors[2usize][0usize],
3303            vectors[3usize][0usize],
3304            vectors[0usize][1usize],
3305            vectors[1usize][1usize],
3306            vectors[2usize][1usize],
3307            vectors[3usize][1usize],
3308            vectors[0usize][2usize],
3309            vectors[1usize][2usize],
3310            vectors[2usize][2usize],
3311            vectors[3usize][2usize],
3312            vectors[0usize][3usize],
3313            vectors[1usize][3usize],
3314            vectors[2usize][3usize],
3315            vectors[3usize][3usize],
3316            vectors[0usize][4usize],
3317            vectors[1usize][4usize],
3318            vectors[2usize][4usize],
3319            vectors[3usize][4usize],
3320            vectors[0usize][5usize],
3321            vectors[1usize][5usize],
3322            vectors[2usize][5usize],
3323            vectors[3usize][5usize],
3324            vectors[0usize][6usize],
3325            vectors[1usize][6usize],
3326            vectors[2usize][6usize],
3327            vectors[3usize][6usize],
3328            vectors[0usize][7usize],
3329            vectors[1usize][7usize],
3330            vectors[2usize][7usize],
3331            vectors[3usize][7usize],
3332        ];
3333    }
3334    #[inline(always)]
3335    fn widen_i16x8(self, a: i16x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
3336        crate::kernel!(
3337            #[inline(always)]
3338            fn kernel(token: Sse2, a: i16x8<Sse2>) -> (i32x4<Sse2>, i32x4<Sse2>) {
3339                let raw = a.into();
3340                let sign = _mm_srai_epi16::<15>(raw);
3341                (
3342                    _mm_unpacklo_epi16(raw, sign).simd_into(token),
3343                    _mm_unpackhi_epi16(raw, sign).simd_into(token),
3344                )
3345            }
3346        );
3347        kernel(self, a)
3348    }
3349    #[inline(always)]
3350    fn widening_mul_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
3351        crate::kernel!(
3352            #[inline(always)]
3353            fn kernel(token: Sse2, a: i16x8<Sse2>, b: i16x8<Sse2>) -> (i32x4<Sse2>, i32x4<Sse2>) {
3354                let a = a.into();
3355                let b = b.into();
3356                let low = _mm_mullo_epi16(a, b);
3357                let high = _mm_mulhi_epi16(a, b);
3358                let result_low = _mm_unpacklo_epi16(low, high);
3359                let result_high = _mm_unpackhi_epi16(low, high);
3360                (result_low.simd_into(token), result_high.simd_into(token))
3361            }
3362        );
3363        kernel(self, a, b)
3364    }
3365    #[inline(always)]
3366    fn widening_add_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
3367        let (a0, a1) = self.widen_i16x8(a);
3368        let (b0, b1) = self.widen_i16x8(b);
3369        (self.add_i32x4(a0, b0), self.add_i32x4(a1, b1))
3370    }
3371    #[inline(always)]
3372    fn pairwise_widening_add_i16x8(self, a: i16x8<Self>) -> i32x4<Self> {
3373        crate::kernel!(
3374            #[inline(always)]
3375            fn kernel(token: Sse2, a: i16x8<Sse2>) -> i32x4<Sse2> {
3376                ({ _mm_madd_epi16(a.into(), _mm_set1_epi16(1)) }).simd_into(token)
3377            }
3378        );
3379        kernel(self, a)
3380    }
3381    #[inline(always)]
3382    fn pairwise_dot_product_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i32x4<Self> {
3383        crate::kernel!(
3384            #[inline(always)]
3385            fn kernel(token: Sse2, a: i16x8<Sse2>, b: i16x8<Sse2>) -> i32x4<Sse2> {
3386                _mm_madd_epi16(a.into(), b.into()).simd_into(token)
3387            }
3388        );
3389        kernel(self, a, b)
3390    }
3391    #[inline(always)]
3392    fn narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
3393        crate::kernel!(
3394            #[inline(always)]
3395            fn kernel(token: Sse2, a: i16x8<Sse2>, b: i16x8<Sse2>) -> i8x16<Sse2> {
3396                let mask = _mm_set1_epi16(0xff);
3397                _mm_packus_epi16(_mm_and_si128(a.into(), mask), _mm_and_si128(b.into(), mask))
3398                    .simd_into(token)
3399            }
3400        );
3401        kernel(self, a, b)
3402    }
3403    #[inline(always)]
3404    fn saturating_narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
3405        crate::kernel!(
3406            #[inline(always)]
3407            fn kernel(token: Sse2, a: i16x8<Sse2>, b: i16x8<Sse2>) -> i8x16<Sse2> {
3408                _mm_packs_epi16(a.into(), b.into()).simd_into(token)
3409            }
3410        );
3411        kernel(self, a, b)
3412    }
3413    #[inline(always)]
3414    fn relaxed_narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
3415        debug_assert!(
3416            a.as_slice()
3417                .iter()
3418                .chain(b.as_slice())
3419                .all(|&value| { value >= i8::MIN as i16 && value <= i8::MAX as i16 }),
3420            "relaxed_narrow inputs must fit in the destination type",
3421        );
3422        self.saturating_narrow_i16x8(a, b)
3423    }
3424    #[inline(always)]
3425    fn splat_u16x8(self, val: u16) -> u16x8<Self> {
3426        crate::kernel!(
3427            #[inline(always)]
3428            fn kernel(token: Sse2, val: u16) -> u16x8<Sse2> {
3429                _mm_set1_epi16(val.cast_signed()).simd_into(token)
3430            }
3431        );
3432        kernel(self, val)
3433    }
3434    #[inline(always)]
3435    fn reverse_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
3436        [
3437            a[7usize], a[6usize], a[5usize], a[4usize], a[3usize], a[2usize], a[1usize], a[0usize],
3438        ]
3439        .simd_into(self)
3440    }
3441    #[inline(always)]
3442    fn slide_u16x8<const SHIFT: usize>(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3443        if SHIFT >= 8usize {
3444            return b;
3445        }
3446        let result = dyn_alignr_128(
3447            self,
3448            Bytes::to_bytes(b).val.0,
3449            Bytes::to_bytes(a).val.0,
3450            SHIFT * 2usize,
3451        );
3452        Bytes::from_bytes(u8x16 {
3453            val: crate::support::Aligned128(result),
3454            simd: self,
3455        })
3456    }
3457    #[inline(always)]
3458    fn compress_u16x8(self, values: u16x8<Self>, mask: mask16x8<Self>) -> u16x8<Self> {
3459        let mask = mask8x16 {
3460            val: crate::transmute::checked_transmute_copy(&mask.val),
3461            simd: self,
3462        };
3463        Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask))
3464    }
3465    #[inline(always)]
3466    fn compress_merge_u16x8(
3467        self,
3468        values: u16x8<Self>,
3469        mask: mask16x8<Self>,
3470        merge: u16x8<Self>,
3471    ) -> u16x8<Self> {
3472        let mask = mask8x16 {
3473            val: crate::transmute::checked_transmute_copy(&mask.val),
3474            simd: self,
3475        };
3476        Bytes::from_bytes(self.compress_merge_u8x16(
3477            Bytes::to_bytes(values),
3478            mask,
3479            Bytes::to_bytes(merge),
3480        ))
3481    }
3482    #[inline(always)]
3483    fn expand_u16x8(self, values: u16x8<Self>, mask: mask16x8<Self>) -> u16x8<Self> {
3484        let mask = mask8x16 {
3485            val: crate::transmute::checked_transmute_copy(&mask.val),
3486            simd: self,
3487        };
3488        Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask))
3489    }
3490    #[inline(always)]
3491    fn expand_merge_u16x8(
3492        self,
3493        values: u16x8<Self>,
3494        mask: mask16x8<Self>,
3495        merge: u16x8<Self>,
3496    ) -> u16x8<Self> {
3497        let mask = mask8x16 {
3498            val: crate::transmute::checked_transmute_copy(&mask.val),
3499            simd: self,
3500        };
3501        Bytes::from_bytes(self.expand_merge_u8x16(
3502            Bytes::to_bytes(values),
3503            mask,
3504            Bytes::to_bytes(merge),
3505        ))
3506    }
3507    #[inline(always)]
3508    fn count_ones_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
3509        [
3510            u16::try_from(a[0usize].count_ones()).unwrap(),
3511            u16::try_from(a[1usize].count_ones()).unwrap(),
3512            u16::try_from(a[2usize].count_ones()).unwrap(),
3513            u16::try_from(a[3usize].count_ones()).unwrap(),
3514            u16::try_from(a[4usize].count_ones()).unwrap(),
3515            u16::try_from(a[5usize].count_ones()).unwrap(),
3516            u16::try_from(a[6usize].count_ones()).unwrap(),
3517            u16::try_from(a[7usize].count_ones()).unwrap(),
3518        ]
3519        .simd_into(self)
3520    }
3521    #[inline(always)]
3522    fn count_zeros_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
3523        [
3524            u16::try_from(a[0usize].count_zeros()).unwrap(),
3525            u16::try_from(a[1usize].count_zeros()).unwrap(),
3526            u16::try_from(a[2usize].count_zeros()).unwrap(),
3527            u16::try_from(a[3usize].count_zeros()).unwrap(),
3528            u16::try_from(a[4usize].count_zeros()).unwrap(),
3529            u16::try_from(a[5usize].count_zeros()).unwrap(),
3530            u16::try_from(a[6usize].count_zeros()).unwrap(),
3531            u16::try_from(a[7usize].count_zeros()).unwrap(),
3532        ]
3533        .simd_into(self)
3534    }
3535    #[inline(always)]
3536    fn add_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3537        crate::kernel!(
3538            #[inline(always)]
3539            fn kernel(token: Sse2, a: u16x8<Sse2>, b: u16x8<Sse2>) -> u16x8<Sse2> {
3540                _mm_add_epi16(a.into(), b.into()).simd_into(token)
3541            }
3542        );
3543        kernel(self, a, b)
3544    }
3545    #[inline(always)]
3546    fn saturating_add_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3547        crate::kernel!(
3548            #[inline(always)]
3549            fn kernel(token: Sse2, a: u16x8<Sse2>, b: u16x8<Sse2>) -> u16x8<Sse2> {
3550                _mm_adds_epu16(a.into(), b.into()).simd_into(token)
3551            }
3552        );
3553        kernel(self, a, b)
3554    }
3555    #[inline(always)]
3556    fn sub_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3557        crate::kernel!(
3558            #[inline(always)]
3559            fn kernel(token: Sse2, a: u16x8<Sse2>, b: u16x8<Sse2>) -> u16x8<Sse2> {
3560                _mm_sub_epi16(a.into(), b.into()).simd_into(token)
3561            }
3562        );
3563        kernel(self, a, b)
3564    }
3565    #[inline(always)]
3566    fn saturating_sub_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3567        crate::kernel!(
3568            #[inline(always)]
3569            fn kernel(token: Sse2, a: u16x8<Sse2>, b: u16x8<Sse2>) -> u16x8<Sse2> {
3570                _mm_subs_epu16(a.into(), b.into()).simd_into(token)
3571            }
3572        );
3573        kernel(self, a, b)
3574    }
3575    #[inline(always)]
3576    fn mul_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3577        crate::kernel!(
3578            #[inline(always)]
3579            fn kernel(token: Sse2, a: u16x8<Sse2>, b: u16x8<Sse2>) -> u16x8<Sse2> {
3580                _mm_mullo_epi16(a.into(), b.into()).simd_into(token)
3581            }
3582        );
3583        kernel(self, a, b)
3584    }
3585    #[inline(always)]
3586    fn and_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3587        crate::kernel!(
3588            #[inline(always)]
3589            fn kernel(token: Sse2, a: u16x8<Sse2>, b: u16x8<Sse2>) -> u16x8<Sse2> {
3590                _mm_and_si128(a.into(), b.into()).simd_into(token)
3591            }
3592        );
3593        kernel(self, a, b)
3594    }
3595    #[inline(always)]
3596    fn or_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3597        crate::kernel!(
3598            #[inline(always)]
3599            fn kernel(token: Sse2, a: u16x8<Sse2>, b: u16x8<Sse2>) -> u16x8<Sse2> {
3600                _mm_or_si128(a.into(), b.into()).simd_into(token)
3601            }
3602        );
3603        kernel(self, a, b)
3604    }
3605    #[inline(always)]
3606    fn xor_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3607        crate::kernel!(
3608            #[inline(always)]
3609            fn kernel(token: Sse2, a: u16x8<Sse2>, b: u16x8<Sse2>) -> u16x8<Sse2> {
3610                _mm_xor_si128(a.into(), b.into()).simd_into(token)
3611            }
3612        );
3613        kernel(self, a, b)
3614    }
3615    #[inline(always)]
3616    fn not_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
3617        a ^ !0
3618    }
3619    #[inline(always)]
3620    fn shl_u16x8(self, a: u16x8<Self>, shift: u32) -> u16x8<Self> {
3621        crate::kernel!(
3622            #[inline(always)]
3623            fn kernel(token: Sse2, a: u16x8<Sse2>, shift: u32) -> u16x8<Sse2> {
3624                _mm_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
3625            }
3626        );
3627        kernel(self, a, shift)
3628    }
3629    #[inline(always)]
3630    fn shlv_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3631        [
3632            u16::wrapping_shl(a[0usize], b[0usize] as u32),
3633            u16::wrapping_shl(a[1usize], b[1usize] as u32),
3634            u16::wrapping_shl(a[2usize], b[2usize] as u32),
3635            u16::wrapping_shl(a[3usize], b[3usize] as u32),
3636            u16::wrapping_shl(a[4usize], b[4usize] as u32),
3637            u16::wrapping_shl(a[5usize], b[5usize] as u32),
3638            u16::wrapping_shl(a[6usize], b[6usize] as u32),
3639            u16::wrapping_shl(a[7usize], b[7usize] as u32),
3640        ]
3641        .simd_into(self)
3642    }
3643    #[inline(always)]
3644    fn shr_u16x8(self, a: u16x8<Self>, shift: u32) -> u16x8<Self> {
3645        crate::kernel!(
3646            #[inline(always)]
3647            fn kernel(token: Sse2, a: u16x8<Sse2>, shift: u32) -> u16x8<Sse2> {
3648                _mm_srl_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
3649            }
3650        );
3651        kernel(self, a, shift)
3652    }
3653    #[inline(always)]
3654    fn shrv_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3655        [
3656            u16::wrapping_shr(a[0usize], b[0usize] as u32),
3657            u16::wrapping_shr(a[1usize], b[1usize] as u32),
3658            u16::wrapping_shr(a[2usize], b[2usize] as u32),
3659            u16::wrapping_shr(a[3usize], b[3usize] as u32),
3660            u16::wrapping_shr(a[4usize], b[4usize] as u32),
3661            u16::wrapping_shr(a[5usize], b[5usize] as u32),
3662            u16::wrapping_shr(a[6usize], b[6usize] as u32),
3663            u16::wrapping_shr(a[7usize], b[7usize] as u32),
3664        ]
3665        .simd_into(self)
3666    }
3667    #[inline(always)]
3668    fn reduce_max_u16x8(self, a: u16x8<Self>) -> u16 {
3669        crate::kernel!(
3670            #[inline(always)]
3671            fn kernel(token: Sse2, a: u16x8<Sse2>) -> u16 {
3672                let reduced: __m128i = a.into();
3673                let shifted = _mm_srli_si128::<8>(reduced);
3674                let reduced = {
3675                    {
3676                        let a = reduced;
3677                        let b = shifted;
3678                        let gt = {
3679                            let sign_bit = _mm_set1_epi16(0x8000u16.cast_signed());
3680                            let lhs_signed = _mm_xor_si128(a, sign_bit);
3681                            let rhs_signed = _mm_xor_si128(b, sign_bit);
3682                            _mm_cmpgt_epi16(lhs_signed, rhs_signed)
3683                        };
3684                        _mm_or_si128(_mm_and_si128(gt, a), _mm_andnot_si128(gt, b))
3685                    }
3686                };
3687                let shifted = _mm_srli_si128::<4>(reduced);
3688                let reduced = {
3689                    {
3690                        let a = reduced;
3691                        let b = shifted;
3692                        let gt = {
3693                            let sign_bit = _mm_set1_epi16(0x8000u16.cast_signed());
3694                            let lhs_signed = _mm_xor_si128(a, sign_bit);
3695                            let rhs_signed = _mm_xor_si128(b, sign_bit);
3696                            _mm_cmpgt_epi16(lhs_signed, rhs_signed)
3697                        };
3698                        _mm_or_si128(_mm_and_si128(gt, a), _mm_andnot_si128(gt, b))
3699                    }
3700                };
3701                let shifted = _mm_srli_si128::<2>(reduced);
3702                let reduced = {
3703                    {
3704                        let a = reduced;
3705                        let b = shifted;
3706                        let gt = {
3707                            let sign_bit = _mm_set1_epi16(0x8000u16.cast_signed());
3708                            let lhs_signed = _mm_xor_si128(a, sign_bit);
3709                            let rhs_signed = _mm_xor_si128(b, sign_bit);
3710                            _mm_cmpgt_epi16(lhs_signed, rhs_signed)
3711                        };
3712                        _mm_or_si128(_mm_and_si128(gt, a), _mm_andnot_si128(gt, b))
3713                    }
3714                };
3715                {
3716                    let lanes: [u16; 8usize] = crate::transmute::checked_transmute_copy(&reduced);
3717                    lanes[0]
3718                }
3719            }
3720        );
3721        kernel(self, a)
3722    }
3723    #[inline(always)]
3724    fn reduce_min_u16x8(self, a: u16x8<Self>) -> u16 {
3725        crate::kernel!(
3726            #[inline(always)]
3727            fn kernel(token: Sse2, a: u16x8<Sse2>) -> u16 {
3728                let reduced: __m128i = a.into();
3729                let shifted = _mm_srli_si128::<8>(reduced);
3730                let reduced = {
3731                    {
3732                        let a = reduced;
3733                        let b = shifted;
3734                        let gt = {
3735                            let sign_bit = _mm_set1_epi16(0x8000u16.cast_signed());
3736                            let lhs_signed = _mm_xor_si128(a, sign_bit);
3737                            let rhs_signed = _mm_xor_si128(b, sign_bit);
3738                            _mm_cmpgt_epi16(lhs_signed, rhs_signed)
3739                        };
3740                        _mm_or_si128(_mm_and_si128(gt, b), _mm_andnot_si128(gt, a))
3741                    }
3742                };
3743                let shifted = _mm_srli_si128::<4>(reduced);
3744                let reduced = {
3745                    {
3746                        let a = reduced;
3747                        let b = shifted;
3748                        let gt = {
3749                            let sign_bit = _mm_set1_epi16(0x8000u16.cast_signed());
3750                            let lhs_signed = _mm_xor_si128(a, sign_bit);
3751                            let rhs_signed = _mm_xor_si128(b, sign_bit);
3752                            _mm_cmpgt_epi16(lhs_signed, rhs_signed)
3753                        };
3754                        _mm_or_si128(_mm_and_si128(gt, b), _mm_andnot_si128(gt, a))
3755                    }
3756                };
3757                let shifted = _mm_srli_si128::<2>(reduced);
3758                let reduced = {
3759                    {
3760                        let a = reduced;
3761                        let b = shifted;
3762                        let gt = {
3763                            let sign_bit = _mm_set1_epi16(0x8000u16.cast_signed());
3764                            let lhs_signed = _mm_xor_si128(a, sign_bit);
3765                            let rhs_signed = _mm_xor_si128(b, sign_bit);
3766                            _mm_cmpgt_epi16(lhs_signed, rhs_signed)
3767                        };
3768                        _mm_or_si128(_mm_and_si128(gt, b), _mm_andnot_si128(gt, a))
3769                    }
3770                };
3771                {
3772                    let lanes: [u16; 8usize] = crate::transmute::checked_transmute_copy(&reduced);
3773                    lanes[0]
3774                }
3775            }
3776        );
3777        kernel(self, a)
3778    }
3779    #[inline(always)]
3780    fn reduce_sum_u16x8(self, a: u16x8<Self>) -> u16 {
3781        crate::kernel!(
3782            #[inline(always)]
3783            fn kernel(token: Sse2, a: u16x8<Sse2>) -> u16 {
3784                let sum: __m128i = a.into();
3785                let sum = _mm_add_epi16(sum, _mm_srli_si128::<8>(sum));
3786                let sum = _mm_add_epi16(sum, _mm_srli_si128::<4>(sum));
3787                let sum = _mm_add_epi16(sum, _mm_srli_si128::<2>(sum));
3788                let lanes: [u16; 8usize] = crate::transmute::checked_transmute_copy(&sum);
3789                lanes[0]
3790            }
3791        );
3792        kernel(self, a)
3793    }
3794    #[inline(always)]
3795    fn reduce_product_u16x8(self, a: u16x8<Self>) -> u16 {
3796        crate::kernel!(
3797            #[inline(always)]
3798            fn kernel(token: Sse2, a: u16x8<Sse2>) -> u16 {
3799                let product: __m128i = a.into();
3800                let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b11_10_11_10>(product));
3801                let product = _mm_mullo_epi16(product, _mm_shuffle_epi32::<0b01_01_01_01>(product));
3802                let product = _mm_mullo_epi16(product, _mm_srli_epi32::<16>(product));
3803                let lanes: [u16; 8usize] = crate::transmute::checked_transmute_copy(&product);
3804                lanes[0]
3805            }
3806        );
3807        kernel(self, a)
3808    }
3809    #[inline(always)]
3810    fn max_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3811        crate::kernel!(
3812            #[inline(always)]
3813            fn kernel(token: Sse2, a: u16x8<Sse2>, b: u16x8<Sse2>) -> u16x8<Sse2> {
3814                {
3815                    let a = a.into();
3816                    let b = b.into();
3817                    let gt = {
3818                        let sign_bit = _mm_set1_epi16(0x8000u16.cast_signed());
3819                        let lhs_signed = _mm_xor_si128(a, sign_bit);
3820                        let rhs_signed = _mm_xor_si128(b, sign_bit);
3821                        _mm_cmpgt_epi16(lhs_signed, rhs_signed)
3822                    };
3823                    _mm_or_si128(_mm_and_si128(gt, a), _mm_andnot_si128(gt, b))
3824                }
3825                .simd_into(token)
3826            }
3827        );
3828        kernel(self, a, b)
3829    }
3830    #[inline(always)]
3831    fn min_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3832        crate::kernel!(
3833            #[inline(always)]
3834            fn kernel(token: Sse2, a: u16x8<Sse2>, b: u16x8<Sse2>) -> u16x8<Sse2> {
3835                {
3836                    let a = a.into();
3837                    let b = b.into();
3838                    let gt = {
3839                        let sign_bit = _mm_set1_epi16(0x8000u16.cast_signed());
3840                        let lhs_signed = _mm_xor_si128(a, sign_bit);
3841                        let rhs_signed = _mm_xor_si128(b, sign_bit);
3842                        _mm_cmpgt_epi16(lhs_signed, rhs_signed)
3843                    };
3844                    _mm_or_si128(_mm_and_si128(gt, b), _mm_andnot_si128(gt, a))
3845                }
3846                .simd_into(token)
3847            }
3848        );
3849        kernel(self, a, b)
3850    }
3851    #[inline(always)]
3852    fn clamp_u16x8(self, a: u16x8<Self>, b: u16x8<Self>, c: u16x8<Self>) -> u16x8<Self> {
3853        debug_assert!(
3854            b.simd_le(c).all_true(),
3855            "clamp requires min <= max for all elements"
3856        );
3857        a.max(b).min(c)
3858    }
3859    #[inline(always)]
3860    fn simd_eq_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
3861        crate::kernel!(
3862            #[inline(always)]
3863            fn kernel(token: Sse2, a: u16x8<Sse2>, b: u16x8<Sse2>) -> mask16x8<Sse2> {
3864                _mm_cmpeq_epi16(a.into(), b.into()).simd_into(token)
3865            }
3866        );
3867        kernel(self, a, b)
3868    }
3869    #[inline(always)]
3870    fn simd_lt_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
3871        crate::kernel!(
3872            #[inline(always)]
3873            fn kernel(token: Sse2, a: u16x8<Sse2>, b: u16x8<Sse2>) -> mask16x8<Sse2> {
3874                {
3875                    let sign_bit = _mm_set1_epi16(0x8000u16.cast_signed());
3876                    let lhs_signed = _mm_xor_si128(b.into(), sign_bit);
3877                    let rhs_signed = _mm_xor_si128(a.into(), sign_bit);
3878                    _mm_cmpgt_epi16(lhs_signed, rhs_signed)
3879                }
3880                .simd_into(token)
3881            }
3882        );
3883        kernel(self, a, b)
3884    }
3885    #[inline(always)]
3886    fn simd_le_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
3887        crate::kernel!(
3888            #[inline(always)]
3889            fn kernel(token: Sse2, a: u16x8<Sse2>, b: u16x8<Sse2>) -> mask16x8<Sse2> {
3890                {
3891                    let all_ones = _mm_set1_epi8(-1);
3892                    _mm_xor_si128(
3893                        {
3894                            let sign_bit = _mm_set1_epi16(0x8000u16.cast_signed());
3895                            let lhs_signed = _mm_xor_si128(a.into(), sign_bit);
3896                            let rhs_signed = _mm_xor_si128(b.into(), sign_bit);
3897                            _mm_cmpgt_epi16(lhs_signed, rhs_signed)
3898                        },
3899                        all_ones,
3900                    )
3901                }
3902                .simd_into(token)
3903            }
3904        );
3905        kernel(self, a, b)
3906    }
3907    #[inline(always)]
3908    fn zip_low_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3909        crate::kernel!(
3910            #[inline(always)]
3911            fn kernel(token: Sse2, a: u16x8<Sse2>, b: u16x8<Sse2>) -> u16x8<Sse2> {
3912                _mm_unpacklo_epi16(a.into(), b.into()).simd_into(token)
3913            }
3914        );
3915        kernel(self, a, b)
3916    }
3917    #[inline(always)]
3918    fn zip_high_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3919        crate::kernel!(
3920            #[inline(always)]
3921            fn kernel(token: Sse2, a: u16x8<Sse2>, b: u16x8<Sse2>) -> u16x8<Sse2> {
3922                _mm_unpackhi_epi16(a.into(), b.into()).simd_into(token)
3923            }
3924        );
3925        kernel(self, a, b)
3926    }
3927    #[inline(always)]
3928    fn unzip_low_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3929        [
3930            a[0usize], a[2usize], a[4usize], a[6usize], b[0usize], b[2usize], b[4usize], b[6usize],
3931        ]
3932        .simd_into(self)
3933    }
3934    #[inline(always)]
3935    fn unzip_high_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
3936        [
3937            a[1usize], a[3usize], a[5usize], a[7usize], b[1usize], b[3usize], b[5usize], b[7usize],
3938        ]
3939        .simd_into(self)
3940    }
3941    #[inline(always)]
3942    fn interleave_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u16x8<Self>, u16x8<Self>) {
3943        (self.zip_low_u16x8(a, b), self.zip_high_u16x8(a, b))
3944    }
3945    #[inline(always)]
3946    fn deinterleave_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u16x8<Self>, u16x8<Self>) {
3947        (self.unzip_low_u16x8(a, b), self.unzip_high_u16x8(a, b))
3948    }
3949    #[inline(always)]
3950    fn select_u16x8(self, a: mask16x8<Self>, b: u16x8<Self>, c: u16x8<Self>) -> u16x8<Self> {
3951        crate::kernel!(
3952            #[inline(always)]
3953            fn kernel(
3954                token: Sse2,
3955                a: mask16x8<Sse2>,
3956                b: u16x8<Sse2>,
3957                c: u16x8<Sse2>,
3958            ) -> u16x8<Sse2> {
3959                _mm_or_si128(
3960                    _mm_and_si128(a.into(), b.into()),
3961                    _mm_andnot_si128(a.into(), c.into()),
3962                )
3963                .simd_into(token)
3964            }
3965        );
3966        kernel(self, a, b, c)
3967    }
3968    #[inline(always)]
3969    fn combine_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x16<Self> {
3970        u16x16 {
3971            val: crate::support::Aligned256([a.val.0, b.val.0]),
3972            simd: self,
3973        }
3974    }
3975    #[inline(always)]
3976    fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8<Self>; 4usize] {
3977        [
3978            [
3979                src[0usize],
3980                src[4usize],
3981                src[8usize],
3982                src[12usize],
3983                src[16usize],
3984                src[20usize],
3985                src[24usize],
3986                src[28usize],
3987            ]
3988            .simd_into(self),
3989            [
3990                src[1usize],
3991                src[5usize],
3992                src[9usize],
3993                src[13usize],
3994                src[17usize],
3995                src[21usize],
3996                src[25usize],
3997                src[29usize],
3998            ]
3999            .simd_into(self),
4000            [
4001                src[2usize],
4002                src[6usize],
4003                src[10usize],
4004                src[14usize],
4005                src[18usize],
4006                src[22usize],
4007                src[26usize],
4008                src[30usize],
4009            ]
4010            .simd_into(self),
4011            [
4012                src[3usize],
4013                src[7usize],
4014                src[11usize],
4015                src[15usize],
4016                src[19usize],
4017                src[23usize],
4018                src[27usize],
4019                src[31usize],
4020            ]
4021            .simd_into(self),
4022        ]
4023    }
4024    #[inline(always)]
4025    fn store_four_interleaved_u16x8(
4026        self,
4027        vectors: [u16x8<Self>; 4usize],
4028        dest: &mut [u16; 32usize],
4029    ) -> () {
4030        *dest = [
4031            vectors[0usize][0usize],
4032            vectors[1usize][0usize],
4033            vectors[2usize][0usize],
4034            vectors[3usize][0usize],
4035            vectors[0usize][1usize],
4036            vectors[1usize][1usize],
4037            vectors[2usize][1usize],
4038            vectors[3usize][1usize],
4039            vectors[0usize][2usize],
4040            vectors[1usize][2usize],
4041            vectors[2usize][2usize],
4042            vectors[3usize][2usize],
4043            vectors[0usize][3usize],
4044            vectors[1usize][3usize],
4045            vectors[2usize][3usize],
4046            vectors[3usize][3usize],
4047            vectors[0usize][4usize],
4048            vectors[1usize][4usize],
4049            vectors[2usize][4usize],
4050            vectors[3usize][4usize],
4051            vectors[0usize][5usize],
4052            vectors[1usize][5usize],
4053            vectors[2usize][5usize],
4054            vectors[3usize][5usize],
4055            vectors[0usize][6usize],
4056            vectors[1usize][6usize],
4057            vectors[2usize][6usize],
4058            vectors[3usize][6usize],
4059            vectors[0usize][7usize],
4060            vectors[1usize][7usize],
4061            vectors[2usize][7usize],
4062            vectors[3usize][7usize],
4063        ];
4064    }
4065    #[inline(always)]
4066    fn widen_u16x8(self, a: u16x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
4067        crate::kernel!(
4068            #[inline(always)]
4069            fn kernel(token: Sse2, a: u16x8<Sse2>) -> (u32x4<Sse2>, u32x4<Sse2>) {
4070                let raw = a.into();
4071                let sign = _mm_setzero_si128();
4072                (
4073                    _mm_unpacklo_epi16(raw, sign).simd_into(token),
4074                    _mm_unpackhi_epi16(raw, sign).simd_into(token),
4075                )
4076            }
4077        );
4078        kernel(self, a)
4079    }
4080    #[inline(always)]
4081    fn widening_mul_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
4082        crate::kernel!(
4083            #[inline(always)]
4084            fn kernel(token: Sse2, a: u16x8<Sse2>, b: u16x8<Sse2>) -> (u32x4<Sse2>, u32x4<Sse2>) {
4085                let a = a.into();
4086                let b = b.into();
4087                let low = _mm_mullo_epi16(a, b);
4088                let high = _mm_mulhi_epu16(a, b);
4089                let result_low = _mm_unpacklo_epi16(low, high);
4090                let result_high = _mm_unpackhi_epi16(low, high);
4091                (result_low.simd_into(token), result_high.simd_into(token))
4092            }
4093        );
4094        kernel(self, a, b)
4095    }
4096    #[inline(always)]
4097    fn widening_add_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
4098        let (a0, a1) = self.widen_u16x8(a);
4099        let (b0, b1) = self.widen_u16x8(b);
4100        (self.add_u32x4(a0, b0), self.add_u32x4(a1, b1))
4101    }
4102    #[inline(always)]
4103    fn pairwise_widening_add_u16x8(self, a: u16x8<Self>) -> u32x4<Self> {
4104        crate::kernel!(
4105            #[inline(always)]
4106            fn kernel(token: Sse2, a: u16x8<Sse2>) -> u32x4<Sse2> {
4107                ({
4108                    let a = a.into();
4109                    _mm_add_epi32(
4110                        _mm_and_si128(a, _mm_set1_epi32(65535)),
4111                        _mm_srli_epi32::<16>(a),
4112                    )
4113                })
4114                .simd_into(token)
4115            }
4116        );
4117        kernel(self, a)
4118    }
4119    #[inline(always)]
4120    fn pairwise_dot_product_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u32x4<Self> {
4121        let (low, high) = self.widening_mul_u16x8(a, b);
4122        let (even, odd) = self.deinterleave_u32x4(low, high);
4123        self.add_u32x4(even, odd)
4124    }
4125    #[inline(always)]
4126    fn narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
4127        crate::kernel!(
4128            #[inline(always)]
4129            fn kernel(token: Sse2, a: u16x8<Sse2>, b: u16x8<Sse2>) -> u8x16<Sse2> {
4130                let mask = _mm_set1_epi16(0xff);
4131                _mm_packus_epi16(_mm_and_si128(a.into(), mask), _mm_and_si128(b.into(), mask))
4132                    .simd_into(token)
4133            }
4134        );
4135        kernel(self, a, b)
4136    }
4137    #[inline(always)]
4138    fn saturating_narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
4139        crate::kernel!(
4140            #[inline(always)]
4141            fn kernel(token: Sse2, a: u16x8<Sse2>, b: u16x8<Sse2>) -> u8x16<Sse2> {
4142                let max = _mm_set1_epi16(u8::MAX as i16);
4143                let a = a.into();
4144                let b = b.into();
4145                let a = _mm_sub_epi16(a, _mm_subs_epu16(a, max));
4146                let b = _mm_sub_epi16(b, _mm_subs_epu16(b, max));
4147                _mm_packus_epi16(a, b).simd_into(token)
4148            }
4149        );
4150        kernel(self, a, b)
4151    }
4152    #[inline(always)]
4153    fn relaxed_narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
4154        crate::kernel!(
4155            #[inline(always)]
4156            fn kernel(token: Sse2, a: u16x8<Sse2>, b: u16x8<Sse2>) -> u8x16<Sse2> {
4157                debug_assert!(
4158                    a.as_slice()
4159                        .iter()
4160                        .chain(b.as_slice())
4161                        .all(|&value| { value >= u8::MIN as u16 && value <= u8::MAX as u16 }),
4162                    "relaxed_narrow inputs must fit in the destination type",
4163                );
4164                _mm_packus_epi16(a.into(), b.into()).simd_into(token)
4165            }
4166        );
4167        kernel(self, a, b)
4168    }
4169    #[inline(always)]
4170    fn splat_mask16x8(self, val: bool) -> mask16x8<Self> {
4171        crate::kernel!(
4172            #[inline(always)]
4173            fn kernel(token: Sse2, val: bool) -> mask16x8<Sse2> {
4174                let val: i16 = if val { !0 } else { 0 };
4175                _mm_set1_epi16(val).simd_into(token)
4176            }
4177        );
4178        kernel(self, val)
4179    }
4180    #[inline(always)]
4181    fn from_bitmask_mask16x8(self, bits: u64) -> mask16x8<Self> {
4182        crate::kernel!(
4183            #[inline(always)]
4184            fn kernel(token: Sse2, bits: u64) -> mask16x8<Sse2> {
4185                {
4186                    let bit_lanes = _mm_set1_epi16(bits as i16);
4187                    let bit_mask = _mm_setr_epi16(1, 2, 4, 8, 16, 32, 64, 128);
4188                    _mm_cmpeq_epi16(_mm_and_si128(bit_lanes, bit_mask), bit_mask)
4189                }
4190                .simd_into(token)
4191            }
4192        );
4193        kernel(self, bits)
4194    }
4195    #[inline(always)]
4196    fn to_bitmask_mask16x8(self, a: mask16x8<Self>) -> u64 {
4197        crate::kernel!(
4198            #[inline(always)]
4199            fn kernel(token: Sse2, a: mask16x8<Sse2>) -> u64 {
4200                {
4201                    let packed = _mm_packs_epi16(a.into(), a.into());
4202                    _mm_movemask_epi8(packed) as u8 as u64
4203                }
4204            }
4205        );
4206        kernel(self, a)
4207    }
4208    #[inline(always)]
4209    fn from_vector_mask16x8(self, vector: i16x8<Self>) -> mask16x8<Self> {
4210        mask16x8 {
4211            val: vector.val,
4212            simd: self,
4213        }
4214    }
4215    #[inline(always)]
4216    fn to_vector_mask16x8(self, a: mask16x8<Self>) -> i16x8<Self> {
4217        i16x8 {
4218            val: a.val,
4219            simd: self,
4220        }
4221    }
4222    #[inline(always)]
4223    fn set_mask16x8(self, a: &mut mask16x8<Self>, index: usize, value: bool) -> () {
4224        assert!(
4225            index < 8usize,
4226            "mask lane index {index} is out of bounds for {} lanes",
4227            8usize
4228        );
4229        let mut lanes: [i16; 8usize] = (*a).into();
4230        lanes[index] = if value { !0 } else { 0 };
4231        *a = lanes.simd_into(self);
4232    }
4233    #[inline(always)]
4234    fn rotate_elements_left_mask16x8<const OFFSET: usize>(
4235        self,
4236        a: mask16x8<Self>,
4237    ) -> mask16x8<Self> {
4238        let int = i16x8 {
4239            val: crate::transmute::checked_transmute_copy(&a.val),
4240            simd: self,
4241        };
4242        let rotated = int.rotate_elements_left::<OFFSET>();
4243        mask16x8 {
4244            val: crate::transmute::checked_transmute_copy(&rotated.val),
4245            simd: self,
4246        }
4247    }
4248    #[inline(always)]
4249    fn rotate_elements_right_mask16x8<const OFFSET: usize>(
4250        self,
4251        a: mask16x8<Self>,
4252    ) -> mask16x8<Self> {
4253        let int = i16x8 {
4254            val: crate::transmute::checked_transmute_copy(&a.val),
4255            simd: self,
4256        };
4257        let rotated = int.rotate_elements_right::<OFFSET>();
4258        mask16x8 {
4259            val: crate::transmute::checked_transmute_copy(&rotated.val),
4260            simd: self,
4261        }
4262    }
4263    #[inline(always)]
4264    fn and_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
4265        crate::kernel!(
4266            #[inline(always)]
4267            fn kernel(token: Sse2, a: mask16x8<Sse2>, b: mask16x8<Sse2>) -> mask16x8<Sse2> {
4268                _mm_and_si128(a.into(), b.into()).simd_into(token)
4269            }
4270        );
4271        kernel(self, a, b)
4272    }
4273    #[inline(always)]
4274    fn or_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
4275        crate::kernel!(
4276            #[inline(always)]
4277            fn kernel(token: Sse2, a: mask16x8<Sse2>, b: mask16x8<Sse2>) -> mask16x8<Sse2> {
4278                _mm_or_si128(a.into(), b.into()).simd_into(token)
4279            }
4280        );
4281        kernel(self, a, b)
4282    }
4283    #[inline(always)]
4284    fn xor_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
4285        crate::kernel!(
4286            #[inline(always)]
4287            fn kernel(token: Sse2, a: mask16x8<Sse2>, b: mask16x8<Sse2>) -> mask16x8<Sse2> {
4288                _mm_xor_si128(a.into(), b.into()).simd_into(token)
4289            }
4290        );
4291        kernel(self, a, b)
4292    }
4293    #[inline(always)]
4294    fn not_mask16x8(self, a: mask16x8<Self>) -> mask16x8<Self> {
4295        self.xor_mask16x8(a, self.splat_mask16x8(true))
4296    }
4297    #[inline(always)]
4298    fn reverse_mask16x8(self, a: mask16x8<Self>) -> mask16x8<Self> {
4299        let lanes = i16x8 {
4300            val: crate::transmute::checked_transmute_copy(&a.val),
4301            simd: self,
4302        };
4303        let reversed = self.reverse_i16x8(lanes);
4304        mask16x8 {
4305            val: crate::transmute::checked_transmute_copy(&reversed.val),
4306            simd: self,
4307        }
4308    }
4309    #[inline(always)]
4310    fn select_mask16x8(
4311        self,
4312        a: mask16x8<Self>,
4313        b: mask16x8<Self>,
4314        c: mask16x8<Self>,
4315    ) -> mask16x8<Self> {
4316        crate::kernel!(
4317            #[inline(always)]
4318            fn kernel(
4319                token: Sse2,
4320                a: mask16x8<Sse2>,
4321                b: mask16x8<Sse2>,
4322                c: mask16x8<Sse2>,
4323            ) -> mask16x8<Sse2> {
4324                _mm_or_si128(
4325                    _mm_and_si128(a.into(), b.into()),
4326                    _mm_andnot_si128(a.into(), c.into()),
4327                )
4328                .simd_into(token)
4329            }
4330        );
4331        kernel(self, a, b, c)
4332    }
4333    #[inline(always)]
4334    fn simd_eq_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
4335        crate::kernel!(
4336            #[inline(always)]
4337            fn kernel(token: Sse2, a: mask16x8<Sse2>, b: mask16x8<Sse2>) -> mask16x8<Sse2> {
4338                _mm_cmpeq_epi16(a.into(), b.into()).simd_into(token)
4339            }
4340        );
4341        kernel(self, a, b)
4342    }
4343    #[inline(always)]
4344    fn any_true_mask16x8(self, a: mask16x8<Self>) -> bool {
4345        crate::kernel!(
4346            #[inline(always)]
4347            fn kernel(token: Sse2, a: mask16x8<Sse2>) -> bool {
4348                _mm_movemask_epi8(a.into()) as u32 != 0
4349            }
4350        );
4351        kernel(self, a)
4352    }
4353    #[inline(always)]
4354    fn all_true_mask16x8(self, a: mask16x8<Self>) -> bool {
4355        crate::kernel!(
4356            #[inline(always)]
4357            fn kernel(token: Sse2, a: mask16x8<Sse2>) -> bool {
4358                _mm_movemask_epi8(a.into()) as u32 == 0xffff
4359            }
4360        );
4361        kernel(self, a)
4362    }
4363    #[inline(always)]
4364    fn any_false_mask16x8(self, a: mask16x8<Self>) -> bool {
4365        crate::kernel!(
4366            #[inline(always)]
4367            fn kernel(token: Sse2, a: mask16x8<Sse2>) -> bool {
4368                _mm_movemask_epi8(a.into()) as u32 != 0xffff
4369            }
4370        );
4371        kernel(self, a)
4372    }
4373    #[inline(always)]
4374    fn all_false_mask16x8(self, a: mask16x8<Self>) -> bool {
4375        crate::kernel!(
4376            #[inline(always)]
4377            fn kernel(token: Sse2, a: mask16x8<Sse2>) -> bool {
4378                _mm_movemask_epi8(a.into()) as u32 == 0
4379            }
4380        );
4381        kernel(self, a)
4382    }
4383    #[inline(always)]
4384    fn combine_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x16<Self> {
4385        mask16x16 {
4386            val: crate::support::Aligned256([a.val.0, b.val.0]),
4387            simd: self,
4388        }
4389    }
4390    #[inline(always)]
4391    fn widen_mask16x8(self, a: mask16x8<Self>) -> (mask32x4<Self>, mask32x4<Self>) {
4392        crate::kernel!(
4393            #[inline(always)]
4394            fn kernel(token: Sse2, a: mask16x8<Sse2>) -> (mask32x4<Sse2>, mask32x4<Sse2>) {
4395                let raw = a.into();
4396                (
4397                    _mm_unpacklo_epi16(raw, raw).simd_into(token),
4398                    _mm_unpackhi_epi16(raw, raw).simd_into(token),
4399                )
4400            }
4401        );
4402        kernel(self, a)
4403    }
4404    #[inline(always)]
4405    fn narrow_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask8x16<Self> {
4406        crate::kernel!(
4407            #[inline(always)]
4408            fn kernel(token: Sse2, a: mask16x8<Sse2>, b: mask16x8<Sse2>) -> mask8x16<Sse2> {
4409                _mm_packs_epi16(a.into(), b.into()).simd_into(token)
4410            }
4411        );
4412        kernel(self, a, b)
4413    }
4414    #[inline(always)]
4415    fn abs_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4416        [
4417            i32::wrapping_abs(a[0usize]),
4418            i32::wrapping_abs(a[1usize]),
4419            i32::wrapping_abs(a[2usize]),
4420            i32::wrapping_abs(a[3usize]),
4421        ]
4422        .simd_into(self)
4423    }
4424    #[inline(always)]
4425    fn splat_i32x4(self, val: i32) -> i32x4<Self> {
4426        crate::kernel!(
4427            #[inline(always)]
4428            fn kernel(token: Sse2, val: i32) -> i32x4<Sse2> {
4429                _mm_set1_epi32(val).simd_into(token)
4430            }
4431        );
4432        kernel(self, val)
4433    }
4434    #[inline(always)]
4435    fn reverse_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4436        [a[3usize], a[2usize], a[1usize], a[0usize]].simd_into(self)
4437    }
4438    #[inline(always)]
4439    fn slide_i32x4<const SHIFT: usize>(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4440        if SHIFT >= 4usize {
4441            return b;
4442        }
4443        let result = dyn_alignr_128(
4444            self,
4445            Bytes::to_bytes(b).val.0,
4446            Bytes::to_bytes(a).val.0,
4447            SHIFT * 4usize,
4448        );
4449        Bytes::from_bytes(u8x16 {
4450            val: crate::support::Aligned128(result),
4451            simd: self,
4452        })
4453    }
4454    #[inline(always)]
4455    fn compress_i32x4(self, values: i32x4<Self>, mask: mask32x4<Self>) -> i32x4<Self> {
4456        let mask = mask8x16 {
4457            val: crate::transmute::checked_transmute_copy(&mask.val),
4458            simd: self,
4459        };
4460        Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask))
4461    }
4462    #[inline(always)]
4463    fn compress_merge_i32x4(
4464        self,
4465        values: i32x4<Self>,
4466        mask: mask32x4<Self>,
4467        merge: i32x4<Self>,
4468    ) -> i32x4<Self> {
4469        let mask = mask8x16 {
4470            val: crate::transmute::checked_transmute_copy(&mask.val),
4471            simd: self,
4472        };
4473        Bytes::from_bytes(self.compress_merge_u8x16(
4474            Bytes::to_bytes(values),
4475            mask,
4476            Bytes::to_bytes(merge),
4477        ))
4478    }
4479    #[inline(always)]
4480    fn expand_i32x4(self, values: i32x4<Self>, mask: mask32x4<Self>) -> i32x4<Self> {
4481        let mask = mask8x16 {
4482            val: crate::transmute::checked_transmute_copy(&mask.val),
4483            simd: self,
4484        };
4485        Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask))
4486    }
4487    #[inline(always)]
4488    fn expand_merge_i32x4(
4489        self,
4490        values: i32x4<Self>,
4491        mask: mask32x4<Self>,
4492        merge: i32x4<Self>,
4493    ) -> i32x4<Self> {
4494        let mask = mask8x16 {
4495            val: crate::transmute::checked_transmute_copy(&mask.val),
4496            simd: self,
4497        };
4498        Bytes::from_bytes(self.expand_merge_u8x16(
4499            Bytes::to_bytes(values),
4500            mask,
4501            Bytes::to_bytes(merge),
4502        ))
4503    }
4504    #[inline(always)]
4505    fn count_ones_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4506        [
4507            a[0usize].count_ones().cast_signed(),
4508            a[1usize].count_ones().cast_signed(),
4509            a[2usize].count_ones().cast_signed(),
4510            a[3usize].count_ones().cast_signed(),
4511        ]
4512        .simd_into(self)
4513    }
4514    #[inline(always)]
4515    fn count_zeros_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4516        [
4517            a[0usize].count_zeros().cast_signed(),
4518            a[1usize].count_zeros().cast_signed(),
4519            a[2usize].count_zeros().cast_signed(),
4520            a[3usize].count_zeros().cast_signed(),
4521        ]
4522        .simd_into(self)
4523    }
4524    #[inline(always)]
4525    fn add_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4526        crate::kernel!(
4527            #[inline(always)]
4528            fn kernel(token: Sse2, a: i32x4<Sse2>, b: i32x4<Sse2>) -> i32x4<Sse2> {
4529                _mm_add_epi32(a.into(), b.into()).simd_into(token)
4530            }
4531        );
4532        kernel(self, a, b)
4533    }
4534    #[inline(always)]
4535    fn saturating_add_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4536        [
4537            i32::saturating_add(a[0usize], b[0usize]),
4538            i32::saturating_add(a[1usize], b[1usize]),
4539            i32::saturating_add(a[2usize], b[2usize]),
4540            i32::saturating_add(a[3usize], b[3usize]),
4541        ]
4542        .simd_into(self)
4543    }
4544    #[inline(always)]
4545    fn sub_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4546        crate::kernel!(
4547            #[inline(always)]
4548            fn kernel(token: Sse2, a: i32x4<Sse2>, b: i32x4<Sse2>) -> i32x4<Sse2> {
4549                _mm_sub_epi32(a.into(), b.into()).simd_into(token)
4550            }
4551        );
4552        kernel(self, a, b)
4553    }
4554    #[inline(always)]
4555    fn saturating_sub_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4556        [
4557            i32::saturating_sub(a[0usize], b[0usize]),
4558            i32::saturating_sub(a[1usize], b[1usize]),
4559            i32::saturating_sub(a[2usize], b[2usize]),
4560            i32::saturating_sub(a[3usize], b[3usize]),
4561        ]
4562        .simd_into(self)
4563    }
4564    #[inline(always)]
4565    fn mul_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4566        [
4567            i32::wrapping_mul(a[0usize], b[0usize]),
4568            i32::wrapping_mul(a[1usize], b[1usize]),
4569            i32::wrapping_mul(a[2usize], b[2usize]),
4570            i32::wrapping_mul(a[3usize], b[3usize]),
4571        ]
4572        .simd_into(self)
4573    }
4574    #[inline(always)]
4575    fn and_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4576        crate::kernel!(
4577            #[inline(always)]
4578            fn kernel(token: Sse2, a: i32x4<Sse2>, b: i32x4<Sse2>) -> i32x4<Sse2> {
4579                _mm_and_si128(a.into(), b.into()).simd_into(token)
4580            }
4581        );
4582        kernel(self, a, b)
4583    }
4584    #[inline(always)]
4585    fn or_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4586        crate::kernel!(
4587            #[inline(always)]
4588            fn kernel(token: Sse2, a: i32x4<Sse2>, b: i32x4<Sse2>) -> i32x4<Sse2> {
4589                _mm_or_si128(a.into(), b.into()).simd_into(token)
4590            }
4591        );
4592        kernel(self, a, b)
4593    }
4594    #[inline(always)]
4595    fn xor_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4596        crate::kernel!(
4597            #[inline(always)]
4598            fn kernel(token: Sse2, a: i32x4<Sse2>, b: i32x4<Sse2>) -> i32x4<Sse2> {
4599                _mm_xor_si128(a.into(), b.into()).simd_into(token)
4600            }
4601        );
4602        kernel(self, a, b)
4603    }
4604    #[inline(always)]
4605    fn not_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4606        a ^ !0
4607    }
4608    #[inline(always)]
4609    fn shl_i32x4(self, a: i32x4<Self>, shift: u32) -> i32x4<Self> {
4610        crate::kernel!(
4611            #[inline(always)]
4612            fn kernel(token: Sse2, a: i32x4<Sse2>, shift: u32) -> i32x4<Sse2> {
4613                _mm_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
4614            }
4615        );
4616        kernel(self, a, shift)
4617    }
4618    #[inline(always)]
4619    fn shlv_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4620        [
4621            i32::wrapping_shl(a[0usize], b[0usize] as u32),
4622            i32::wrapping_shl(a[1usize], b[1usize] as u32),
4623            i32::wrapping_shl(a[2usize], b[2usize] as u32),
4624            i32::wrapping_shl(a[3usize], b[3usize] as u32),
4625        ]
4626        .simd_into(self)
4627    }
4628    #[inline(always)]
4629    fn shr_i32x4(self, a: i32x4<Self>, shift: u32) -> i32x4<Self> {
4630        crate::kernel!(
4631            #[inline(always)]
4632            fn kernel(token: Sse2, a: i32x4<Sse2>, shift: u32) -> i32x4<Sse2> {
4633                _mm_sra_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
4634            }
4635        );
4636        kernel(self, a, shift)
4637    }
4638    #[inline(always)]
4639    fn shrv_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4640        [
4641            i32::wrapping_shr(a[0usize], b[0usize] as u32),
4642            i32::wrapping_shr(a[1usize], b[1usize] as u32),
4643            i32::wrapping_shr(a[2usize], b[2usize] as u32),
4644            i32::wrapping_shr(a[3usize], b[3usize] as u32),
4645        ]
4646        .simd_into(self)
4647    }
4648    #[inline(always)]
4649    fn reduce_max_i32x4(self, a: i32x4<Self>) -> i32 {
4650        crate::kernel!(
4651            #[inline(always)]
4652            fn kernel(token: Sse2, a: i32x4<Sse2>) -> i32 {
4653                let reduced: __m128i = a.into();
4654                let shifted = _mm_srli_si128::<8>(reduced);
4655                let reduced = {
4656                    {
4657                        let a = reduced;
4658                        let b = shifted;
4659                        let gt = _mm_cmpgt_epi32(a, b);
4660                        _mm_or_si128(_mm_and_si128(gt, a), _mm_andnot_si128(gt, b))
4661                    }
4662                };
4663                let shifted = _mm_srli_si128::<4>(reduced);
4664                let reduced = {
4665                    {
4666                        let a = reduced;
4667                        let b = shifted;
4668                        let gt = _mm_cmpgt_epi32(a, b);
4669                        _mm_or_si128(_mm_and_si128(gt, a), _mm_andnot_si128(gt, b))
4670                    }
4671                };
4672                {
4673                    let lanes: [i32; 4usize] = crate::transmute::checked_transmute_copy(&reduced);
4674                    lanes[0]
4675                }
4676            }
4677        );
4678        kernel(self, a)
4679    }
4680    #[inline(always)]
4681    fn reduce_min_i32x4(self, a: i32x4<Self>) -> i32 {
4682        crate::kernel!(
4683            #[inline(always)]
4684            fn kernel(token: Sse2, a: i32x4<Sse2>) -> i32 {
4685                let reduced: __m128i = a.into();
4686                let shifted = _mm_srli_si128::<8>(reduced);
4687                let reduced = {
4688                    {
4689                        let a = reduced;
4690                        let b = shifted;
4691                        let gt = _mm_cmpgt_epi32(a, b);
4692                        _mm_or_si128(_mm_and_si128(gt, b), _mm_andnot_si128(gt, a))
4693                    }
4694                };
4695                let shifted = _mm_srli_si128::<4>(reduced);
4696                let reduced = {
4697                    {
4698                        let a = reduced;
4699                        let b = shifted;
4700                        let gt = _mm_cmpgt_epi32(a, b);
4701                        _mm_or_si128(_mm_and_si128(gt, b), _mm_andnot_si128(gt, a))
4702                    }
4703                };
4704                {
4705                    let lanes: [i32; 4usize] = crate::transmute::checked_transmute_copy(&reduced);
4706                    lanes[0]
4707                }
4708            }
4709        );
4710        kernel(self, a)
4711    }
4712    #[inline(always)]
4713    fn reduce_sum_i32x4(self, a: i32x4<Self>) -> i32 {
4714        crate::kernel!(
4715            #[inline(always)]
4716            fn kernel(token: Sse2, a: i32x4<Sse2>) -> i32 {
4717                let sum: __m128i = a.into();
4718                let sum = _mm_add_epi32(sum, _mm_srli_si128::<8>(sum));
4719                let sum = _mm_add_epi32(sum, _mm_srli_si128::<4>(sum));
4720                let lanes: [i32; 4usize] = crate::transmute::checked_transmute_copy(&sum);
4721                lanes[0]
4722            }
4723        );
4724        kernel(self, a)
4725    }
4726    #[inline(always)]
4727    fn reduce_product_i32x4(self, a: i32x4<Self>) -> i32 {
4728        let mul_level_0: [i32; 2usize] = [
4729            a[0usize].wrapping_mul(a[1usize]),
4730            a[2usize].wrapping_mul(a[3usize]),
4731        ];
4732        let mul_level_1: [i32; 1usize] = [mul_level_0[0usize].wrapping_mul(mul_level_0[1usize])];
4733        mul_level_1[0]
4734    }
4735    #[inline(always)]
4736    fn max_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4737        crate::kernel!(
4738            #[inline(always)]
4739            fn kernel(token: Sse2, a: i32x4<Sse2>, b: i32x4<Sse2>) -> i32x4<Sse2> {
4740                {
4741                    let a = a.into();
4742                    let b = b.into();
4743                    let gt = _mm_cmpgt_epi32(a, b);
4744                    _mm_or_si128(_mm_and_si128(gt, a), _mm_andnot_si128(gt, b))
4745                }
4746                .simd_into(token)
4747            }
4748        );
4749        kernel(self, a, b)
4750    }
4751    #[inline(always)]
4752    fn min_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4753        crate::kernel!(
4754            #[inline(always)]
4755            fn kernel(token: Sse2, a: i32x4<Sse2>, b: i32x4<Sse2>) -> i32x4<Sse2> {
4756                {
4757                    let a = a.into();
4758                    let b = b.into();
4759                    let gt = _mm_cmpgt_epi32(a, b);
4760                    _mm_or_si128(_mm_and_si128(gt, b), _mm_andnot_si128(gt, a))
4761                }
4762                .simd_into(token)
4763            }
4764        );
4765        kernel(self, a, b)
4766    }
4767    #[inline(always)]
4768    fn clamp_i32x4(self, a: i32x4<Self>, b: i32x4<Self>, c: i32x4<Self>) -> i32x4<Self> {
4769        debug_assert!(
4770            b.simd_le(c).all_true(),
4771            "clamp requires min <= max for all elements"
4772        );
4773        a.max(b).min(c)
4774    }
4775    #[inline(always)]
4776    fn simd_eq_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
4777        crate::kernel!(
4778            #[inline(always)]
4779            fn kernel(token: Sse2, a: i32x4<Sse2>, b: i32x4<Sse2>) -> mask32x4<Sse2> {
4780                _mm_cmpeq_epi32(a.into(), b.into()).simd_into(token)
4781            }
4782        );
4783        kernel(self, a, b)
4784    }
4785    #[inline(always)]
4786    fn simd_lt_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
4787        crate::kernel!(
4788            #[inline(always)]
4789            fn kernel(token: Sse2, a: i32x4<Sse2>, b: i32x4<Sse2>) -> mask32x4<Sse2> {
4790                _mm_cmpgt_epi32(b.into(), a.into()).simd_into(token)
4791            }
4792        );
4793        kernel(self, a, b)
4794    }
4795    #[inline(always)]
4796    fn simd_le_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
4797        crate::kernel!(
4798            #[inline(always)]
4799            fn kernel(token: Sse2, a: i32x4<Sse2>, b: i32x4<Sse2>) -> mask32x4<Sse2> {
4800                {
4801                    let all_ones = _mm_set1_epi8(-1);
4802                    _mm_xor_si128(_mm_cmpgt_epi32(a.into(), b.into()), all_ones)
4803                }
4804                .simd_into(token)
4805            }
4806        );
4807        kernel(self, a, b)
4808    }
4809    #[inline(always)]
4810    fn zip_low_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4811        crate::kernel!(
4812            #[inline(always)]
4813            fn kernel(token: Sse2, a: i32x4<Sse2>, b: i32x4<Sse2>) -> i32x4<Sse2> {
4814                _mm_unpacklo_epi32(a.into(), b.into()).simd_into(token)
4815            }
4816        );
4817        kernel(self, a, b)
4818    }
4819    #[inline(always)]
4820    fn zip_high_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4821        crate::kernel!(
4822            #[inline(always)]
4823            fn kernel(token: Sse2, a: i32x4<Sse2>, b: i32x4<Sse2>) -> i32x4<Sse2> {
4824                _mm_unpackhi_epi32(a.into(), b.into()).simd_into(token)
4825            }
4826        );
4827        kernel(self, a, b)
4828    }
4829    #[inline(always)]
4830    fn unzip_low_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4831        crate::kernel!(
4832            #[inline(always)]
4833            fn kernel(token: Sse2, a: i32x4<Sse2>, b: i32x4<Sse2>) -> i32x4<Sse2> {
4834                let t1 = _mm_shuffle_epi32::<0b11_01_10_00>(a.into());
4835                let t2 = _mm_shuffle_epi32::<0b11_01_10_00>(b.into());
4836                _mm_unpacklo_epi64(t1, t2).simd_into(token)
4837            }
4838        );
4839        kernel(self, a, b)
4840    }
4841    #[inline(always)]
4842    fn unzip_high_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
4843        crate::kernel!(
4844            #[inline(always)]
4845            fn kernel(token: Sse2, a: i32x4<Sse2>, b: i32x4<Sse2>) -> i32x4<Sse2> {
4846                let t1 = _mm_shuffle_epi32::<0b11_01_10_00>(a.into());
4847                let t2 = _mm_shuffle_epi32::<0b11_01_10_00>(b.into());
4848                _mm_unpackhi_epi64(t1, t2).simd_into(token)
4849            }
4850        );
4851        kernel(self, a, b)
4852    }
4853    #[inline(always)]
4854    fn interleave_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i32x4<Self>, i32x4<Self>) {
4855        (self.zip_low_i32x4(a, b), self.zip_high_i32x4(a, b))
4856    }
4857    #[inline(always)]
4858    fn deinterleave_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i32x4<Self>, i32x4<Self>) {
4859        (self.unzip_low_i32x4(a, b), self.unzip_high_i32x4(a, b))
4860    }
4861    #[inline(always)]
4862    fn select_i32x4(self, a: mask32x4<Self>, b: i32x4<Self>, c: i32x4<Self>) -> i32x4<Self> {
4863        crate::kernel!(
4864            #[inline(always)]
4865            fn kernel(
4866                token: Sse2,
4867                a: mask32x4<Sse2>,
4868                b: i32x4<Sse2>,
4869                c: i32x4<Sse2>,
4870            ) -> i32x4<Sse2> {
4871                _mm_or_si128(
4872                    _mm_and_si128(a.into(), b.into()),
4873                    _mm_andnot_si128(a.into(), c.into()),
4874                )
4875                .simd_into(token)
4876            }
4877        );
4878        kernel(self, a, b, c)
4879    }
4880    #[inline(always)]
4881    fn combine_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x8<Self> {
4882        i32x8 {
4883            val: crate::support::Aligned256([a.val.0, b.val.0]),
4884            simd: self,
4885        }
4886    }
4887    #[inline(always)]
4888    fn neg_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
4889        crate::kernel!(
4890            #[inline(always)]
4891            fn kernel(token: Sse2, a: i32x4<Sse2>) -> i32x4<Sse2> {
4892                _mm_sub_epi32(_mm_setzero_si128(), a.into()).simd_into(token)
4893            }
4894        );
4895        kernel(self, a)
4896    }
4897    #[inline(always)]
4898    fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4<Self>; 4usize] {
4899        crate::kernel!(
4900            #[inline(always)]
4901            fn kernel(token: Sse2, src: &[i32; 16usize]) -> [i32x4<Sse2>; 4usize] {
4902                let (chunks, []) = src.as_chunks::<4usize>() else {
4903                    unreachable!()
4904                };
4905                let v0: __m128i =
4906                    crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[0]);
4907                let v1: __m128i =
4908                    crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[1]);
4909                let v2: __m128i =
4910                    crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[2]);
4911                let v3: __m128i =
4912                    crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[3]);
4913                let tmp0 = _mm_unpacklo_epi32(v0, v1);
4914                let tmp1 = _mm_unpackhi_epi32(v0, v1);
4915                let tmp2 = _mm_unpacklo_epi32(v2, v3);
4916                let tmp3 = _mm_unpackhi_epi32(v2, v3);
4917                let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
4918                let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
4919                let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
4920                let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
4921                [
4922                    out0.simd_into(token),
4923                    out1.simd_into(token),
4924                    out2.simd_into(token),
4925                    out3.simd_into(token),
4926                ]
4927            }
4928        );
4929        kernel(self, src)
4930    }
4931    #[inline(always)]
4932    fn store_four_interleaved_i32x4(
4933        self,
4934        vectors: [i32x4<Self>; 4usize],
4935        dest: &mut [i32; 16usize],
4936    ) -> () {
4937        crate::kernel!(
4938            #[inline(always)]
4939            fn kernel(
4940                token: Sse2,
4941                vectors: [i32x4<Sse2>; 4usize],
4942                dest: &mut [i32; 16usize],
4943            ) -> () {
4944                let _ = token;
4945                let v0: __m128i = vectors[0].into();
4946                let v1: __m128i = vectors[1].into();
4947                let v2: __m128i = vectors[2].into();
4948                let v3: __m128i = vectors[3].into();
4949                let tmp0 = _mm_unpacklo_epi32(v0, v1);
4950                let tmp1 = _mm_unpackhi_epi32(v0, v1);
4951                let tmp2 = _mm_unpacklo_epi32(v2, v3);
4952                let tmp3 = _mm_unpackhi_epi32(v2, v3);
4953                let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
4954                let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
4955                let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
4956                let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
4957                let (chunks, []) = dest.as_chunks_mut::<4usize>() else {
4958                    unreachable!()
4959                };
4960                crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>(
4961                    out0,
4962                    &mut chunks[0],
4963                );
4964                crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>(
4965                    out1,
4966                    &mut chunks[1],
4967                );
4968                crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>(
4969                    out2,
4970                    &mut chunks[2],
4971                );
4972                crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>(
4973                    out3,
4974                    &mut chunks[3],
4975                );
4976            }
4977        );
4978        kernel(self, vectors, dest);
4979    }
4980    #[inline(always)]
4981    fn widen_i32x4(self, a: i32x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
4982        crate::kernel!(
4983            #[inline(always)]
4984            fn kernel(token: Sse2, a: i32x4<Sse2>) -> (i64x2<Sse2>, i64x2<Sse2>) {
4985                let raw = a.into();
4986                let sign = _mm_srai_epi32::<31>(raw);
4987                (
4988                    _mm_unpacklo_epi32(raw, sign).simd_into(token),
4989                    _mm_unpackhi_epi32(raw, sign).simd_into(token),
4990                )
4991            }
4992        );
4993        kernel(self, a)
4994    }
4995    #[inline(always)]
4996    fn widening_mul_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
4997        let (a0, a1) = self.widen_i32x4(a);
4998        let (b0, b1) = self.widen_i32x4(b);
4999        (self.mul_i64x2(a0, b0), self.mul_i64x2(a1, b1))
5000    }
5001    #[inline(always)]
5002    fn widening_add_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
5003        let (a0, a1) = self.widen_i32x4(a);
5004        let (b0, b1) = self.widen_i32x4(b);
5005        (self.add_i64x2(a0, b0), self.add_i64x2(a1, b1))
5006    }
5007    #[inline(always)]
5008    fn pairwise_widening_add_i32x4(self, a: i32x4<Self>) -> i64x2<Self> {
5009        crate::kernel!(
5010            #[inline(always)]
5011            fn kernel(token: Sse2, a: i32x4<Sse2>) -> i64x2<Sse2> {
5012                ({
5013                    let y = _mm_xor_si128(a.into(), _mm_set1_epi32(i32::MIN));
5014                    _mm_add_epi64(
5015                        _mm_or_si128(y, _mm_set1_epi64x(-0x1_0000_0000)),
5016                        _mm_srli_epi64::<32>(y),
5017                    )
5018                })
5019                .simd_into(token)
5020            }
5021        );
5022        kernel(self, a)
5023    }
5024    #[inline(always)]
5025    fn pairwise_dot_product_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i64x2<Self> {
5026        let (low, high) = self.widening_mul_i32x4(a, b);
5027        let (even, odd) = self.deinterleave_i64x2(low, high);
5028        self.add_i64x2(even, odd)
5029    }
5030    #[inline(always)]
5031    fn narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
5032        [
5033            a[0usize] as i16,
5034            a[1usize] as i16,
5035            a[2usize] as i16,
5036            a[3usize] as i16,
5037            b[0usize] as i16,
5038            b[1usize] as i16,
5039            b[2usize] as i16,
5040            b[3usize] as i16,
5041        ]
5042        .simd_into(self)
5043    }
5044    #[inline(always)]
5045    fn saturating_narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
5046        crate::kernel!(
5047            #[inline(always)]
5048            fn kernel(token: Sse2, a: i32x4<Sse2>, b: i32x4<Sse2>) -> i16x8<Sse2> {
5049                _mm_packs_epi32(a.into(), b.into()).simd_into(token)
5050            }
5051        );
5052        kernel(self, a, b)
5053    }
5054    #[inline(always)]
5055    fn relaxed_narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
5056        debug_assert!(
5057            a.as_slice()
5058                .iter()
5059                .chain(b.as_slice())
5060                .all(|&value| { value >= i16::MIN as i32 && value <= i16::MAX as i32 }),
5061            "relaxed_narrow inputs must fit in the destination type",
5062        );
5063        self.saturating_narrow_i32x4(a, b)
5064    }
5065    #[inline(always)]
5066    fn cvt_f32_i32x4(self, a: i32x4<Self>) -> f32x4<Self> {
5067        crate::kernel!(
5068            #[inline(always)]
5069            fn kernel(token: Sse2, a: i32x4<Sse2>) -> f32x4<Sse2> {
5070                _mm_cvtepi32_ps(a.into()).simd_into(token)
5071            }
5072        );
5073        kernel(self, a)
5074    }
5075    #[inline(always)]
5076    fn splat_u32x4(self, val: u32) -> u32x4<Self> {
5077        crate::kernel!(
5078            #[inline(always)]
5079            fn kernel(token: Sse2, val: u32) -> u32x4<Sse2> {
5080                _mm_set1_epi32(val.cast_signed()).simd_into(token)
5081            }
5082        );
5083        kernel(self, val)
5084    }
5085    #[inline(always)]
5086    fn reverse_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
5087        [a[3usize], a[2usize], a[1usize], a[0usize]].simd_into(self)
5088    }
5089    #[inline(always)]
5090    fn slide_u32x4<const SHIFT: usize>(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5091        if SHIFT >= 4usize {
5092            return b;
5093        }
5094        let result = dyn_alignr_128(
5095            self,
5096            Bytes::to_bytes(b).val.0,
5097            Bytes::to_bytes(a).val.0,
5098            SHIFT * 4usize,
5099        );
5100        Bytes::from_bytes(u8x16 {
5101            val: crate::support::Aligned128(result),
5102            simd: self,
5103        })
5104    }
5105    #[inline(always)]
5106    fn compress_u32x4(self, values: u32x4<Self>, mask: mask32x4<Self>) -> u32x4<Self> {
5107        let mask = mask8x16 {
5108            val: crate::transmute::checked_transmute_copy(&mask.val),
5109            simd: self,
5110        };
5111        Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask))
5112    }
5113    #[inline(always)]
5114    fn compress_merge_u32x4(
5115        self,
5116        values: u32x4<Self>,
5117        mask: mask32x4<Self>,
5118        merge: u32x4<Self>,
5119    ) -> u32x4<Self> {
5120        let mask = mask8x16 {
5121            val: crate::transmute::checked_transmute_copy(&mask.val),
5122            simd: self,
5123        };
5124        Bytes::from_bytes(self.compress_merge_u8x16(
5125            Bytes::to_bytes(values),
5126            mask,
5127            Bytes::to_bytes(merge),
5128        ))
5129    }
5130    #[inline(always)]
5131    fn expand_u32x4(self, values: u32x4<Self>, mask: mask32x4<Self>) -> u32x4<Self> {
5132        let mask = mask8x16 {
5133            val: crate::transmute::checked_transmute_copy(&mask.val),
5134            simd: self,
5135        };
5136        Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask))
5137    }
5138    #[inline(always)]
5139    fn expand_merge_u32x4(
5140        self,
5141        values: u32x4<Self>,
5142        mask: mask32x4<Self>,
5143        merge: u32x4<Self>,
5144    ) -> u32x4<Self> {
5145        let mask = mask8x16 {
5146            val: crate::transmute::checked_transmute_copy(&mask.val),
5147            simd: self,
5148        };
5149        Bytes::from_bytes(self.expand_merge_u8x16(
5150            Bytes::to_bytes(values),
5151            mask,
5152            Bytes::to_bytes(merge),
5153        ))
5154    }
5155    #[inline(always)]
5156    fn count_ones_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
5157        [
5158            a[0usize].count_ones(),
5159            a[1usize].count_ones(),
5160            a[2usize].count_ones(),
5161            a[3usize].count_ones(),
5162        ]
5163        .simd_into(self)
5164    }
5165    #[inline(always)]
5166    fn count_zeros_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
5167        [
5168            a[0usize].count_zeros(),
5169            a[1usize].count_zeros(),
5170            a[2usize].count_zeros(),
5171            a[3usize].count_zeros(),
5172        ]
5173        .simd_into(self)
5174    }
5175    #[inline(always)]
5176    fn add_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5177        crate::kernel!(
5178            #[inline(always)]
5179            fn kernel(token: Sse2, a: u32x4<Sse2>, b: u32x4<Sse2>) -> u32x4<Sse2> {
5180                _mm_add_epi32(a.into(), b.into()).simd_into(token)
5181            }
5182        );
5183        kernel(self, a, b)
5184    }
5185    #[inline(always)]
5186    fn saturating_add_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5187        [
5188            u32::saturating_add(a[0usize], b[0usize]),
5189            u32::saturating_add(a[1usize], b[1usize]),
5190            u32::saturating_add(a[2usize], b[2usize]),
5191            u32::saturating_add(a[3usize], b[3usize]),
5192        ]
5193        .simd_into(self)
5194    }
5195    #[inline(always)]
5196    fn sub_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5197        crate::kernel!(
5198            #[inline(always)]
5199            fn kernel(token: Sse2, a: u32x4<Sse2>, b: u32x4<Sse2>) -> u32x4<Sse2> {
5200                _mm_sub_epi32(a.into(), b.into()).simd_into(token)
5201            }
5202        );
5203        kernel(self, a, b)
5204    }
5205    #[inline(always)]
5206    fn saturating_sub_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5207        [
5208            u32::saturating_sub(a[0usize], b[0usize]),
5209            u32::saturating_sub(a[1usize], b[1usize]),
5210            u32::saturating_sub(a[2usize], b[2usize]),
5211            u32::saturating_sub(a[3usize], b[3usize]),
5212        ]
5213        .simd_into(self)
5214    }
5215    #[inline(always)]
5216    fn mul_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5217        [
5218            u32::wrapping_mul(a[0usize], b[0usize]),
5219            u32::wrapping_mul(a[1usize], b[1usize]),
5220            u32::wrapping_mul(a[2usize], b[2usize]),
5221            u32::wrapping_mul(a[3usize], b[3usize]),
5222        ]
5223        .simd_into(self)
5224    }
5225    #[inline(always)]
5226    fn and_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5227        crate::kernel!(
5228            #[inline(always)]
5229            fn kernel(token: Sse2, a: u32x4<Sse2>, b: u32x4<Sse2>) -> u32x4<Sse2> {
5230                _mm_and_si128(a.into(), b.into()).simd_into(token)
5231            }
5232        );
5233        kernel(self, a, b)
5234    }
5235    #[inline(always)]
5236    fn or_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5237        crate::kernel!(
5238            #[inline(always)]
5239            fn kernel(token: Sse2, a: u32x4<Sse2>, b: u32x4<Sse2>) -> u32x4<Sse2> {
5240                _mm_or_si128(a.into(), b.into()).simd_into(token)
5241            }
5242        );
5243        kernel(self, a, b)
5244    }
5245    #[inline(always)]
5246    fn xor_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5247        crate::kernel!(
5248            #[inline(always)]
5249            fn kernel(token: Sse2, a: u32x4<Sse2>, b: u32x4<Sse2>) -> u32x4<Sse2> {
5250                _mm_xor_si128(a.into(), b.into()).simd_into(token)
5251            }
5252        );
5253        kernel(self, a, b)
5254    }
5255    #[inline(always)]
5256    fn not_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
5257        a ^ !0
5258    }
5259    #[inline(always)]
5260    fn shl_u32x4(self, a: u32x4<Self>, shift: u32) -> u32x4<Self> {
5261        crate::kernel!(
5262            #[inline(always)]
5263            fn kernel(token: Sse2, a: u32x4<Sse2>, shift: u32) -> u32x4<Sse2> {
5264                _mm_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
5265            }
5266        );
5267        kernel(self, a, shift)
5268    }
5269    #[inline(always)]
5270    fn shlv_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5271        [
5272            u32::wrapping_shl(a[0usize], b[0usize] as u32),
5273            u32::wrapping_shl(a[1usize], b[1usize] as u32),
5274            u32::wrapping_shl(a[2usize], b[2usize] as u32),
5275            u32::wrapping_shl(a[3usize], b[3usize] as u32),
5276        ]
5277        .simd_into(self)
5278    }
5279    #[inline(always)]
5280    fn shr_u32x4(self, a: u32x4<Self>, shift: u32) -> u32x4<Self> {
5281        crate::kernel!(
5282            #[inline(always)]
5283            fn kernel(token: Sse2, a: u32x4<Sse2>, shift: u32) -> u32x4<Sse2> {
5284                _mm_srl_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
5285            }
5286        );
5287        kernel(self, a, shift)
5288    }
5289    #[inline(always)]
5290    fn shrv_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5291        [
5292            u32::wrapping_shr(a[0usize], b[0usize] as u32),
5293            u32::wrapping_shr(a[1usize], b[1usize] as u32),
5294            u32::wrapping_shr(a[2usize], b[2usize] as u32),
5295            u32::wrapping_shr(a[3usize], b[3usize] as u32),
5296        ]
5297        .simd_into(self)
5298    }
5299    #[inline(always)]
5300    fn reduce_max_u32x4(self, a: u32x4<Self>) -> u32 {
5301        crate::kernel!(
5302            #[inline(always)]
5303            fn kernel(token: Sse2, a: u32x4<Sse2>) -> u32 {
5304                let reduced: __m128i = a.into();
5305                let shifted = _mm_srli_si128::<8>(reduced);
5306                let reduced = {
5307                    {
5308                        let a = reduced;
5309                        let b = shifted;
5310                        let gt = {
5311                            let sign_bit = _mm_set1_epi32(0x80000000u32.cast_signed());
5312                            let lhs_signed = _mm_xor_si128(a, sign_bit);
5313                            let rhs_signed = _mm_xor_si128(b, sign_bit);
5314                            _mm_cmpgt_epi32(lhs_signed, rhs_signed)
5315                        };
5316                        _mm_or_si128(_mm_and_si128(gt, a), _mm_andnot_si128(gt, b))
5317                    }
5318                };
5319                let shifted = _mm_srli_si128::<4>(reduced);
5320                let reduced = {
5321                    {
5322                        let a = reduced;
5323                        let b = shifted;
5324                        let gt = {
5325                            let sign_bit = _mm_set1_epi32(0x80000000u32.cast_signed());
5326                            let lhs_signed = _mm_xor_si128(a, sign_bit);
5327                            let rhs_signed = _mm_xor_si128(b, sign_bit);
5328                            _mm_cmpgt_epi32(lhs_signed, rhs_signed)
5329                        };
5330                        _mm_or_si128(_mm_and_si128(gt, a), _mm_andnot_si128(gt, b))
5331                    }
5332                };
5333                {
5334                    let lanes: [u32; 4usize] = crate::transmute::checked_transmute_copy(&reduced);
5335                    lanes[0]
5336                }
5337            }
5338        );
5339        kernel(self, a)
5340    }
5341    #[inline(always)]
5342    fn reduce_min_u32x4(self, a: u32x4<Self>) -> u32 {
5343        crate::kernel!(
5344            #[inline(always)]
5345            fn kernel(token: Sse2, a: u32x4<Sse2>) -> u32 {
5346                let reduced: __m128i = a.into();
5347                let shifted = _mm_srli_si128::<8>(reduced);
5348                let reduced = {
5349                    {
5350                        let a = reduced;
5351                        let b = shifted;
5352                        let gt = {
5353                            let sign_bit = _mm_set1_epi32(0x80000000u32.cast_signed());
5354                            let lhs_signed = _mm_xor_si128(a, sign_bit);
5355                            let rhs_signed = _mm_xor_si128(b, sign_bit);
5356                            _mm_cmpgt_epi32(lhs_signed, rhs_signed)
5357                        };
5358                        _mm_or_si128(_mm_and_si128(gt, b), _mm_andnot_si128(gt, a))
5359                    }
5360                };
5361                let shifted = _mm_srli_si128::<4>(reduced);
5362                let reduced = {
5363                    {
5364                        let a = reduced;
5365                        let b = shifted;
5366                        let gt = {
5367                            let sign_bit = _mm_set1_epi32(0x80000000u32.cast_signed());
5368                            let lhs_signed = _mm_xor_si128(a, sign_bit);
5369                            let rhs_signed = _mm_xor_si128(b, sign_bit);
5370                            _mm_cmpgt_epi32(lhs_signed, rhs_signed)
5371                        };
5372                        _mm_or_si128(_mm_and_si128(gt, b), _mm_andnot_si128(gt, a))
5373                    }
5374                };
5375                {
5376                    let lanes: [u32; 4usize] = crate::transmute::checked_transmute_copy(&reduced);
5377                    lanes[0]
5378                }
5379            }
5380        );
5381        kernel(self, a)
5382    }
5383    #[inline(always)]
5384    fn reduce_sum_u32x4(self, a: u32x4<Self>) -> u32 {
5385        crate::kernel!(
5386            #[inline(always)]
5387            fn kernel(token: Sse2, a: u32x4<Sse2>) -> u32 {
5388                let sum: __m128i = a.into();
5389                let sum = _mm_add_epi32(sum, _mm_srli_si128::<8>(sum));
5390                let sum = _mm_add_epi32(sum, _mm_srli_si128::<4>(sum));
5391                let lanes: [u32; 4usize] = crate::transmute::checked_transmute_copy(&sum);
5392                lanes[0]
5393            }
5394        );
5395        kernel(self, a)
5396    }
5397    #[inline(always)]
5398    fn reduce_product_u32x4(self, a: u32x4<Self>) -> u32 {
5399        let mul_level_0: [u32; 2usize] = [
5400            a[0usize].wrapping_mul(a[1usize]),
5401            a[2usize].wrapping_mul(a[3usize]),
5402        ];
5403        let mul_level_1: [u32; 1usize] = [mul_level_0[0usize].wrapping_mul(mul_level_0[1usize])];
5404        mul_level_1[0]
5405    }
5406    #[inline(always)]
5407    fn max_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5408        crate::kernel!(
5409            #[inline(always)]
5410            fn kernel(token: Sse2, a: u32x4<Sse2>, b: u32x4<Sse2>) -> u32x4<Sse2> {
5411                {
5412                    let a = a.into();
5413                    let b = b.into();
5414                    let gt = {
5415                        let sign_bit = _mm_set1_epi32(0x80000000u32.cast_signed());
5416                        let lhs_signed = _mm_xor_si128(a, sign_bit);
5417                        let rhs_signed = _mm_xor_si128(b, sign_bit);
5418                        _mm_cmpgt_epi32(lhs_signed, rhs_signed)
5419                    };
5420                    _mm_or_si128(_mm_and_si128(gt, a), _mm_andnot_si128(gt, b))
5421                }
5422                .simd_into(token)
5423            }
5424        );
5425        kernel(self, a, b)
5426    }
5427    #[inline(always)]
5428    fn min_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5429        crate::kernel!(
5430            #[inline(always)]
5431            fn kernel(token: Sse2, a: u32x4<Sse2>, b: u32x4<Sse2>) -> u32x4<Sse2> {
5432                {
5433                    let a = a.into();
5434                    let b = b.into();
5435                    let gt = {
5436                        let sign_bit = _mm_set1_epi32(0x80000000u32.cast_signed());
5437                        let lhs_signed = _mm_xor_si128(a, sign_bit);
5438                        let rhs_signed = _mm_xor_si128(b, sign_bit);
5439                        _mm_cmpgt_epi32(lhs_signed, rhs_signed)
5440                    };
5441                    _mm_or_si128(_mm_and_si128(gt, b), _mm_andnot_si128(gt, a))
5442                }
5443                .simd_into(token)
5444            }
5445        );
5446        kernel(self, a, b)
5447    }
5448    #[inline(always)]
5449    fn clamp_u32x4(self, a: u32x4<Self>, b: u32x4<Self>, c: u32x4<Self>) -> u32x4<Self> {
5450        debug_assert!(
5451            b.simd_le(c).all_true(),
5452            "clamp requires min <= max for all elements"
5453        );
5454        a.max(b).min(c)
5455    }
5456    #[inline(always)]
5457    fn simd_eq_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
5458        crate::kernel!(
5459            #[inline(always)]
5460            fn kernel(token: Sse2, a: u32x4<Sse2>, b: u32x4<Sse2>) -> mask32x4<Sse2> {
5461                _mm_cmpeq_epi32(a.into(), b.into()).simd_into(token)
5462            }
5463        );
5464        kernel(self, a, b)
5465    }
5466    #[inline(always)]
5467    fn simd_lt_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
5468        crate::kernel!(
5469            #[inline(always)]
5470            fn kernel(token: Sse2, a: u32x4<Sse2>, b: u32x4<Sse2>) -> mask32x4<Sse2> {
5471                {
5472                    let sign_bit = _mm_set1_epi32(0x80000000u32.cast_signed());
5473                    let lhs_signed = _mm_xor_si128(b.into(), sign_bit);
5474                    let rhs_signed = _mm_xor_si128(a.into(), sign_bit);
5475                    _mm_cmpgt_epi32(lhs_signed, rhs_signed)
5476                }
5477                .simd_into(token)
5478            }
5479        );
5480        kernel(self, a, b)
5481    }
5482    #[inline(always)]
5483    fn simd_le_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
5484        crate::kernel!(
5485            #[inline(always)]
5486            fn kernel(token: Sse2, a: u32x4<Sse2>, b: u32x4<Sse2>) -> mask32x4<Sse2> {
5487                {
5488                    let all_ones = _mm_set1_epi8(-1);
5489                    _mm_xor_si128(
5490                        {
5491                            let sign_bit = _mm_set1_epi32(0x80000000u32.cast_signed());
5492                            let lhs_signed = _mm_xor_si128(a.into(), sign_bit);
5493                            let rhs_signed = _mm_xor_si128(b.into(), sign_bit);
5494                            _mm_cmpgt_epi32(lhs_signed, rhs_signed)
5495                        },
5496                        all_ones,
5497                    )
5498                }
5499                .simd_into(token)
5500            }
5501        );
5502        kernel(self, a, b)
5503    }
5504    #[inline(always)]
5505    fn zip_low_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5506        crate::kernel!(
5507            #[inline(always)]
5508            fn kernel(token: Sse2, a: u32x4<Sse2>, b: u32x4<Sse2>) -> u32x4<Sse2> {
5509                _mm_unpacklo_epi32(a.into(), b.into()).simd_into(token)
5510            }
5511        );
5512        kernel(self, a, b)
5513    }
5514    #[inline(always)]
5515    fn zip_high_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5516        crate::kernel!(
5517            #[inline(always)]
5518            fn kernel(token: Sse2, a: u32x4<Sse2>, b: u32x4<Sse2>) -> u32x4<Sse2> {
5519                _mm_unpackhi_epi32(a.into(), b.into()).simd_into(token)
5520            }
5521        );
5522        kernel(self, a, b)
5523    }
5524    #[inline(always)]
5525    fn unzip_low_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5526        crate::kernel!(
5527            #[inline(always)]
5528            fn kernel(token: Sse2, a: u32x4<Sse2>, b: u32x4<Sse2>) -> u32x4<Sse2> {
5529                let t1 = _mm_shuffle_epi32::<0b11_01_10_00>(a.into());
5530                let t2 = _mm_shuffle_epi32::<0b11_01_10_00>(b.into());
5531                _mm_unpacklo_epi64(t1, t2).simd_into(token)
5532            }
5533        );
5534        kernel(self, a, b)
5535    }
5536    #[inline(always)]
5537    fn unzip_high_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
5538        crate::kernel!(
5539            #[inline(always)]
5540            fn kernel(token: Sse2, a: u32x4<Sse2>, b: u32x4<Sse2>) -> u32x4<Sse2> {
5541                let t1 = _mm_shuffle_epi32::<0b11_01_10_00>(a.into());
5542                let t2 = _mm_shuffle_epi32::<0b11_01_10_00>(b.into());
5543                _mm_unpackhi_epi64(t1, t2).simd_into(token)
5544            }
5545        );
5546        kernel(self, a, b)
5547    }
5548    #[inline(always)]
5549    fn interleave_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u32x4<Self>, u32x4<Self>) {
5550        (self.zip_low_u32x4(a, b), self.zip_high_u32x4(a, b))
5551    }
5552    #[inline(always)]
5553    fn deinterleave_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u32x4<Self>, u32x4<Self>) {
5554        (self.unzip_low_u32x4(a, b), self.unzip_high_u32x4(a, b))
5555    }
5556    #[inline(always)]
5557    fn select_u32x4(self, a: mask32x4<Self>, b: u32x4<Self>, c: u32x4<Self>) -> u32x4<Self> {
5558        crate::kernel!(
5559            #[inline(always)]
5560            fn kernel(
5561                token: Sse2,
5562                a: mask32x4<Sse2>,
5563                b: u32x4<Sse2>,
5564                c: u32x4<Sse2>,
5565            ) -> u32x4<Sse2> {
5566                _mm_or_si128(
5567                    _mm_and_si128(a.into(), b.into()),
5568                    _mm_andnot_si128(a.into(), c.into()),
5569                )
5570                .simd_into(token)
5571            }
5572        );
5573        kernel(self, a, b, c)
5574    }
5575    #[inline(always)]
5576    fn combine_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x8<Self> {
5577        u32x8 {
5578            val: crate::support::Aligned256([a.val.0, b.val.0]),
5579            simd: self,
5580        }
5581    }
5582    #[inline(always)]
5583    fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4<Self>; 4usize] {
5584        crate::kernel!(
5585            #[inline(always)]
5586            fn kernel(token: Sse2, src: &[u32; 16usize]) -> [u32x4<Sse2>; 4usize] {
5587                let (chunks, []) = src.as_chunks::<4usize>() else {
5588                    unreachable!()
5589                };
5590                let v0: __m128i =
5591                    crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]);
5592                let v1: __m128i =
5593                    crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]);
5594                let v2: __m128i =
5595                    crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]);
5596                let v3: __m128i =
5597                    crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]);
5598                let tmp0 = _mm_unpacklo_epi32(v0, v1);
5599                let tmp1 = _mm_unpackhi_epi32(v0, v1);
5600                let tmp2 = _mm_unpacklo_epi32(v2, v3);
5601                let tmp3 = _mm_unpackhi_epi32(v2, v3);
5602                let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
5603                let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
5604                let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
5605                let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
5606                [
5607                    out0.simd_into(token),
5608                    out1.simd_into(token),
5609                    out2.simd_into(token),
5610                    out3.simd_into(token),
5611                ]
5612            }
5613        );
5614        kernel(self, src)
5615    }
5616    #[inline(always)]
5617    fn store_four_interleaved_u32x4(
5618        self,
5619        vectors: [u32x4<Self>; 4usize],
5620        dest: &mut [u32; 16usize],
5621    ) -> () {
5622        crate::kernel!(
5623            #[inline(always)]
5624            fn kernel(
5625                token: Sse2,
5626                vectors: [u32x4<Sse2>; 4usize],
5627                dest: &mut [u32; 16usize],
5628            ) -> () {
5629                let _ = token;
5630                let v0: __m128i = vectors[0].into();
5631                let v1: __m128i = vectors[1].into();
5632                let v2: __m128i = vectors[2].into();
5633                let v3: __m128i = vectors[3].into();
5634                let tmp0 = _mm_unpacklo_epi32(v0, v1);
5635                let tmp1 = _mm_unpackhi_epi32(v0, v1);
5636                let tmp2 = _mm_unpacklo_epi32(v2, v3);
5637                let tmp3 = _mm_unpackhi_epi32(v2, v3);
5638                let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
5639                let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
5640                let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
5641                let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
5642                let (chunks, []) = dest.as_chunks_mut::<4usize>() else {
5643                    unreachable!()
5644                };
5645                crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>(
5646                    out0,
5647                    &mut chunks[0],
5648                );
5649                crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>(
5650                    out1,
5651                    &mut chunks[1],
5652                );
5653                crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>(
5654                    out2,
5655                    &mut chunks[2],
5656                );
5657                crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>(
5658                    out3,
5659                    &mut chunks[3],
5660                );
5661            }
5662        );
5663        kernel(self, vectors, dest);
5664    }
5665    #[inline(always)]
5666    fn widen_u32x4(self, a: u32x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
5667        crate::kernel!(
5668            #[inline(always)]
5669            fn kernel(token: Sse2, a: u32x4<Sse2>) -> (u64x2<Sse2>, u64x2<Sse2>) {
5670                let raw = a.into();
5671                let sign = _mm_setzero_si128();
5672                (
5673                    _mm_unpacklo_epi32(raw, sign).simd_into(token),
5674                    _mm_unpackhi_epi32(raw, sign).simd_into(token),
5675                )
5676            }
5677        );
5678        kernel(self, a)
5679    }
5680    #[inline(always)]
5681    fn widening_mul_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
5682        let (a0, a1) = self.widen_u32x4(a);
5683        let (b0, b1) = self.widen_u32x4(b);
5684        (self.mul_u64x2(a0, b0), self.mul_u64x2(a1, b1))
5685    }
5686    #[inline(always)]
5687    fn widening_add_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
5688        let (a0, a1) = self.widen_u32x4(a);
5689        let (b0, b1) = self.widen_u32x4(b);
5690        (self.add_u64x2(a0, b0), self.add_u64x2(a1, b1))
5691    }
5692    #[inline(always)]
5693    fn pairwise_widening_add_u32x4(self, a: u32x4<Self>) -> u64x2<Self> {
5694        crate::kernel!(
5695            #[inline(always)]
5696            fn kernel(token: Sse2, a: u32x4<Sse2>) -> u64x2<Sse2> {
5697                ({
5698                    let a = a.into();
5699                    _mm_add_epi64(
5700                        _mm_and_si128(a, _mm_set1_epi64x(4294967295)),
5701                        _mm_srli_epi64::<32>(a),
5702                    )
5703                })
5704                .simd_into(token)
5705            }
5706        );
5707        kernel(self, a)
5708    }
5709    #[inline(always)]
5710    fn pairwise_dot_product_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u64x2<Self> {
5711        let (low, high) = self.widening_mul_u32x4(a, b);
5712        let (even, odd) = self.deinterleave_u64x2(low, high);
5713        self.add_u64x2(even, odd)
5714    }
5715    #[inline(always)]
5716    fn narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
5717        [
5718            a[0usize] as u16,
5719            a[1usize] as u16,
5720            a[2usize] as u16,
5721            a[3usize] as u16,
5722            b[0usize] as u16,
5723            b[1usize] as u16,
5724            b[2usize] as u16,
5725            b[3usize] as u16,
5726        ]
5727        .simd_into(self)
5728    }
5729    #[inline(always)]
5730    fn saturating_narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
5731        [
5732            a[0usize].clamp(u16::MIN as u32, u16::MAX as u32) as u16,
5733            a[1usize].clamp(u16::MIN as u32, u16::MAX as u32) as u16,
5734            a[2usize].clamp(u16::MIN as u32, u16::MAX as u32) as u16,
5735            a[3usize].clamp(u16::MIN as u32, u16::MAX as u32) as u16,
5736            b[0usize].clamp(u16::MIN as u32, u16::MAX as u32) as u16,
5737            b[1usize].clamp(u16::MIN as u32, u16::MAX as u32) as u16,
5738            b[2usize].clamp(u16::MIN as u32, u16::MAX as u32) as u16,
5739            b[3usize].clamp(u16::MIN as u32, u16::MAX as u32) as u16,
5740        ]
5741        .simd_into(self)
5742    }
5743    #[inline(always)]
5744    fn relaxed_narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
5745        debug_assert!(
5746            a.as_slice()
5747                .iter()
5748                .chain(b.as_slice())
5749                .all(|&value| { value >= u16::MIN as u32 && value <= u16::MAX as u32 }),
5750            "relaxed_narrow inputs must fit in the destination type",
5751        );
5752        self.narrow_u32x4(a, b)
5753    }
5754    #[inline(always)]
5755    fn cvt_f32_u32x4(self, a: u32x4<Self>) -> f32x4<Self> {
5756        [
5757            a[0usize] as f32,
5758            a[1usize] as f32,
5759            a[2usize] as f32,
5760            a[3usize] as f32,
5761        ]
5762        .simd_into(self)
5763    }
5764    #[inline(always)]
5765    fn splat_mask32x4(self, val: bool) -> mask32x4<Self> {
5766        crate::kernel!(
5767            #[inline(always)]
5768            fn kernel(token: Sse2, val: bool) -> mask32x4<Sse2> {
5769                let val: i32 = if val { !0 } else { 0 };
5770                _mm_set1_epi32(val).simd_into(token)
5771            }
5772        );
5773        kernel(self, val)
5774    }
5775    #[inline(always)]
5776    fn from_bitmask_mask32x4(self, bits: u64) -> mask32x4<Self> {
5777        crate::kernel!(
5778            #[inline(always)]
5779            fn kernel(token: Sse2, bits: u64) -> mask32x4<Sse2> {
5780                {
5781                    let bit_lanes = _mm_set1_epi32(bits as i32);
5782                    let bit_mask = _mm_setr_epi32(1, 2, 4, 8);
5783                    _mm_cmpeq_epi32(_mm_and_si128(bit_lanes, bit_mask), bit_mask)
5784                }
5785                .simd_into(token)
5786            }
5787        );
5788        kernel(self, bits)
5789    }
5790    #[inline(always)]
5791    fn to_bitmask_mask32x4(self, a: mask32x4<Self>) -> u64 {
5792        crate::kernel!(
5793            #[inline(always)]
5794            fn kernel(token: Sse2, a: mask32x4<Sse2>) -> u64 {
5795                _mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 as u64
5796            }
5797        );
5798        kernel(self, a)
5799    }
5800    #[inline(always)]
5801    fn from_vector_mask32x4(self, vector: i32x4<Self>) -> mask32x4<Self> {
5802        mask32x4 {
5803            val: vector.val,
5804            simd: self,
5805        }
5806    }
5807    #[inline(always)]
5808    fn to_vector_mask32x4(self, a: mask32x4<Self>) -> i32x4<Self> {
5809        i32x4 {
5810            val: a.val,
5811            simd: self,
5812        }
5813    }
5814    #[inline(always)]
5815    fn set_mask32x4(self, a: &mut mask32x4<Self>, index: usize, value: bool) -> () {
5816        assert!(
5817            index < 4usize,
5818            "mask lane index {index} is out of bounds for {} lanes",
5819            4usize
5820        );
5821        let mut lanes: [i32; 4usize] = (*a).into();
5822        lanes[index] = if value { !0 } else { 0 };
5823        *a = lanes.simd_into(self);
5824    }
5825    #[inline(always)]
5826    fn rotate_elements_left_mask32x4<const OFFSET: usize>(
5827        self,
5828        a: mask32x4<Self>,
5829    ) -> mask32x4<Self> {
5830        let int = i32x4 {
5831            val: crate::transmute::checked_transmute_copy(&a.val),
5832            simd: self,
5833        };
5834        let rotated = int.rotate_elements_left::<OFFSET>();
5835        mask32x4 {
5836            val: crate::transmute::checked_transmute_copy(&rotated.val),
5837            simd: self,
5838        }
5839    }
5840    #[inline(always)]
5841    fn rotate_elements_right_mask32x4<const OFFSET: usize>(
5842        self,
5843        a: mask32x4<Self>,
5844    ) -> mask32x4<Self> {
5845        let int = i32x4 {
5846            val: crate::transmute::checked_transmute_copy(&a.val),
5847            simd: self,
5848        };
5849        let rotated = int.rotate_elements_right::<OFFSET>();
5850        mask32x4 {
5851            val: crate::transmute::checked_transmute_copy(&rotated.val),
5852            simd: self,
5853        }
5854    }
5855    #[inline(always)]
5856    fn and_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
5857        crate::kernel!(
5858            #[inline(always)]
5859            fn kernel(token: Sse2, a: mask32x4<Sse2>, b: mask32x4<Sse2>) -> mask32x4<Sse2> {
5860                _mm_and_si128(a.into(), b.into()).simd_into(token)
5861            }
5862        );
5863        kernel(self, a, b)
5864    }
5865    #[inline(always)]
5866    fn or_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
5867        crate::kernel!(
5868            #[inline(always)]
5869            fn kernel(token: Sse2, a: mask32x4<Sse2>, b: mask32x4<Sse2>) -> mask32x4<Sse2> {
5870                _mm_or_si128(a.into(), b.into()).simd_into(token)
5871            }
5872        );
5873        kernel(self, a, b)
5874    }
5875    #[inline(always)]
5876    fn xor_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
5877        crate::kernel!(
5878            #[inline(always)]
5879            fn kernel(token: Sse2, a: mask32x4<Sse2>, b: mask32x4<Sse2>) -> mask32x4<Sse2> {
5880                _mm_xor_si128(a.into(), b.into()).simd_into(token)
5881            }
5882        );
5883        kernel(self, a, b)
5884    }
5885    #[inline(always)]
5886    fn not_mask32x4(self, a: mask32x4<Self>) -> mask32x4<Self> {
5887        self.xor_mask32x4(a, self.splat_mask32x4(true))
5888    }
5889    #[inline(always)]
5890    fn reverse_mask32x4(self, a: mask32x4<Self>) -> mask32x4<Self> {
5891        let lanes = i32x4 {
5892            val: crate::transmute::checked_transmute_copy(&a.val),
5893            simd: self,
5894        };
5895        let reversed = self.reverse_i32x4(lanes);
5896        mask32x4 {
5897            val: crate::transmute::checked_transmute_copy(&reversed.val),
5898            simd: self,
5899        }
5900    }
5901    #[inline(always)]
5902    fn select_mask32x4(
5903        self,
5904        a: mask32x4<Self>,
5905        b: mask32x4<Self>,
5906        c: mask32x4<Self>,
5907    ) -> mask32x4<Self> {
5908        crate::kernel!(
5909            #[inline(always)]
5910            fn kernel(
5911                token: Sse2,
5912                a: mask32x4<Sse2>,
5913                b: mask32x4<Sse2>,
5914                c: mask32x4<Sse2>,
5915            ) -> mask32x4<Sse2> {
5916                _mm_or_si128(
5917                    _mm_and_si128(a.into(), b.into()),
5918                    _mm_andnot_si128(a.into(), c.into()),
5919                )
5920                .simd_into(token)
5921            }
5922        );
5923        kernel(self, a, b, c)
5924    }
5925    #[inline(always)]
5926    fn simd_eq_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
5927        crate::kernel!(
5928            #[inline(always)]
5929            fn kernel(token: Sse2, a: mask32x4<Sse2>, b: mask32x4<Sse2>) -> mask32x4<Sse2> {
5930                _mm_cmpeq_epi32(a.into(), b.into()).simd_into(token)
5931            }
5932        );
5933        kernel(self, a, b)
5934    }
5935    #[inline(always)]
5936    fn any_true_mask32x4(self, a: mask32x4<Self>) -> bool {
5937        crate::kernel!(
5938            #[inline(always)]
5939            fn kernel(token: Sse2, a: mask32x4<Sse2>) -> bool {
5940                _mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 != 0
5941            }
5942        );
5943        kernel(self, a)
5944    }
5945    #[inline(always)]
5946    fn all_true_mask32x4(self, a: mask32x4<Self>) -> bool {
5947        crate::kernel!(
5948            #[inline(always)]
5949            fn kernel(token: Sse2, a: mask32x4<Sse2>) -> bool {
5950                _mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 == 0b1111
5951            }
5952        );
5953        kernel(self, a)
5954    }
5955    #[inline(always)]
5956    fn any_false_mask32x4(self, a: mask32x4<Self>) -> bool {
5957        crate::kernel!(
5958            #[inline(always)]
5959            fn kernel(token: Sse2, a: mask32x4<Sse2>) -> bool {
5960                _mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 != 0b1111
5961            }
5962        );
5963        kernel(self, a)
5964    }
5965    #[inline(always)]
5966    fn all_false_mask32x4(self, a: mask32x4<Self>) -> bool {
5967        crate::kernel!(
5968            #[inline(always)]
5969            fn kernel(token: Sse2, a: mask32x4<Sse2>) -> bool {
5970                _mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 == 0
5971            }
5972        );
5973        kernel(self, a)
5974    }
5975    #[inline(always)]
5976    fn combine_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x8<Self> {
5977        mask32x8 {
5978            val: crate::support::Aligned256([a.val.0, b.val.0]),
5979            simd: self,
5980        }
5981    }
5982    #[inline(always)]
5983    fn widen_mask32x4(self, a: mask32x4<Self>) -> (mask64x2<Self>, mask64x2<Self>) {
5984        crate::kernel!(
5985            #[inline(always)]
5986            fn kernel(token: Sse2, a: mask32x4<Sse2>) -> (mask64x2<Sse2>, mask64x2<Sse2>) {
5987                let raw = a.into();
5988                (
5989                    _mm_unpacklo_epi32(raw, raw).simd_into(token),
5990                    _mm_unpackhi_epi32(raw, raw).simd_into(token),
5991                )
5992            }
5993        );
5994        kernel(self, a)
5995    }
5996    #[inline(always)]
5997    fn narrow_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask16x8<Self> {
5998        crate::kernel!(
5999            #[inline(always)]
6000            fn kernel(token: Sse2, a: mask32x4<Sse2>, b: mask32x4<Sse2>) -> mask16x8<Sse2> {
6001                _mm_packs_epi32(a.into(), b.into()).simd_into(token)
6002            }
6003        );
6004        kernel(self, a, b)
6005    }
6006    #[inline(always)]
6007    fn abs_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6008        crate::kernel!(
6009            #[inline(always)]
6010            fn kernel(token: Sse2, a: f64x2<Sse2>) -> f64x2<Sse2> {
6011                _mm_andnot_pd(_mm_set1_pd(-0.0), a.into()).simd_into(token)
6012            }
6013        );
6014        kernel(self, a)
6015    }
6016    #[inline(always)]
6017    fn splat_f64x2(self, val: f64) -> f64x2<Self> {
6018        crate::kernel!(
6019            #[inline(always)]
6020            fn kernel(token: Sse2, val: f64) -> f64x2<Sse2> {
6021                _mm_set1_pd(val).simd_into(token)
6022            }
6023        );
6024        kernel(self, val)
6025    }
6026    #[inline(always)]
6027    fn reverse_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6028        [a[1usize], a[0usize]].simd_into(self)
6029    }
6030    #[inline(always)]
6031    fn slide_f64x2<const SHIFT: usize>(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6032        if SHIFT >= 2usize {
6033            return b;
6034        }
6035        let result = dyn_alignr_128(
6036            self,
6037            Bytes::to_bytes(b).val.0,
6038            Bytes::to_bytes(a).val.0,
6039            SHIFT * 8usize,
6040        );
6041        Bytes::from_bytes(u8x16 {
6042            val: crate::support::Aligned128(result),
6043            simd: self,
6044        })
6045    }
6046    #[inline(always)]
6047    fn compress_f64x2(self, values: f64x2<Self>, mask: mask64x2<Self>) -> f64x2<Self> {
6048        let mask = mask8x16 {
6049            val: crate::transmute::checked_transmute_copy(&mask.val),
6050            simd: self,
6051        };
6052        Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask))
6053    }
6054    #[inline(always)]
6055    fn compress_merge_f64x2(
6056        self,
6057        values: f64x2<Self>,
6058        mask: mask64x2<Self>,
6059        merge: f64x2<Self>,
6060    ) -> f64x2<Self> {
6061        let mask = mask8x16 {
6062            val: crate::transmute::checked_transmute_copy(&mask.val),
6063            simd: self,
6064        };
6065        Bytes::from_bytes(self.compress_merge_u8x16(
6066            Bytes::to_bytes(values),
6067            mask,
6068            Bytes::to_bytes(merge),
6069        ))
6070    }
6071    #[inline(always)]
6072    fn expand_f64x2(self, values: f64x2<Self>, mask: mask64x2<Self>) -> f64x2<Self> {
6073        let mask = mask8x16 {
6074            val: crate::transmute::checked_transmute_copy(&mask.val),
6075            simd: self,
6076        };
6077        Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask))
6078    }
6079    #[inline(always)]
6080    fn expand_merge_f64x2(
6081        self,
6082        values: f64x2<Self>,
6083        mask: mask64x2<Self>,
6084        merge: f64x2<Self>,
6085    ) -> f64x2<Self> {
6086        let mask = mask8x16 {
6087            val: crate::transmute::checked_transmute_copy(&mask.val),
6088            simd: self,
6089        };
6090        Bytes::from_bytes(self.expand_merge_u8x16(
6091            Bytes::to_bytes(values),
6092            mask,
6093            Bytes::to_bytes(merge),
6094        ))
6095    }
6096    #[inline(always)]
6097    fn neg_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6098        crate::kernel!(
6099            #[inline(always)]
6100            fn kernel(token: Sse2, a: f64x2<Sse2>) -> f64x2<Sse2> {
6101                _mm_xor_pd(a.into(), _mm_set1_pd(-0.0)).simd_into(token)
6102            }
6103        );
6104        kernel(self, a)
6105    }
6106    #[inline(always)]
6107    fn sqrt_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6108        crate::kernel!(
6109            #[inline(always)]
6110            fn kernel(token: Sse2, a: f64x2<Sse2>) -> f64x2<Sse2> {
6111                _mm_sqrt_pd(a.into()).simd_into(token)
6112            }
6113        );
6114        kernel(self, a)
6115    }
6116    #[inline(always)]
6117    fn approximate_recip_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6118        1.0 / a
6119    }
6120    #[inline(always)]
6121    fn add_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6122        crate::kernel!(
6123            #[inline(always)]
6124            fn kernel(token: Sse2, a: f64x2<Sse2>, b: f64x2<Sse2>) -> f64x2<Sse2> {
6125                _mm_add_pd(a.into(), b.into()).simd_into(token)
6126            }
6127        );
6128        kernel(self, a, b)
6129    }
6130    #[inline(always)]
6131    fn sub_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6132        crate::kernel!(
6133            #[inline(always)]
6134            fn kernel(token: Sse2, a: f64x2<Sse2>, b: f64x2<Sse2>) -> f64x2<Sse2> {
6135                _mm_sub_pd(a.into(), b.into()).simd_into(token)
6136            }
6137        );
6138        kernel(self, a, b)
6139    }
6140    #[inline(always)]
6141    fn mul_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6142        crate::kernel!(
6143            #[inline(always)]
6144            fn kernel(token: Sse2, a: f64x2<Sse2>, b: f64x2<Sse2>) -> f64x2<Sse2> {
6145                _mm_mul_pd(a.into(), b.into()).simd_into(token)
6146            }
6147        );
6148        kernel(self, a, b)
6149    }
6150    #[inline(always)]
6151    fn div_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6152        crate::kernel!(
6153            #[inline(always)]
6154            fn kernel(token: Sse2, a: f64x2<Sse2>, b: f64x2<Sse2>) -> f64x2<Sse2> {
6155                _mm_div_pd(a.into(), b.into()).simd_into(token)
6156            }
6157        );
6158        kernel(self, a, b)
6159    }
6160    #[inline(always)]
6161    fn copysign_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6162        crate::kernel!(
6163            #[inline(always)]
6164            fn kernel(token: Sse2, a: f64x2<Sse2>, b: f64x2<Sse2>) -> f64x2<Sse2> {
6165                let mask = _mm_set1_pd(-0.0);
6166                _mm_or_pd(_mm_and_pd(mask, b.into()), _mm_andnot_pd(mask, a.into()))
6167                    .simd_into(token)
6168            }
6169        );
6170        kernel(self, a, b)
6171    }
6172    #[inline(always)]
6173    fn reduce_max_f64x2(self, a: f64x2<Self>) -> f64 {
6174        crate::kernel!(
6175            #[inline(always)]
6176            fn kernel(token: Sse2, a: f64x2<Sse2>) -> f64 {
6177                let reduced: __m128d = a.into();
6178                let shifted = _mm_castsi128_pd(_mm_srli_si128::<8>(_mm_castpd_si128(reduced)));
6179                let reduced = { _mm_max_pd(reduced, shifted) };
6180                _mm_cvtsd_f64(reduced)
6181            }
6182        );
6183        kernel(self, a)
6184    }
6185    #[inline(always)]
6186    fn reduce_min_f64x2(self, a: f64x2<Self>) -> f64 {
6187        crate::kernel!(
6188            #[inline(always)]
6189            fn kernel(token: Sse2, a: f64x2<Sse2>) -> f64 {
6190                let reduced: __m128d = a.into();
6191                let shifted = _mm_castsi128_pd(_mm_srli_si128::<8>(_mm_castpd_si128(reduced)));
6192                let reduced = { _mm_min_pd(reduced, shifted) };
6193                _mm_cvtsd_f64(reduced)
6194            }
6195        );
6196        kernel(self, a)
6197    }
6198    #[inline(always)]
6199    fn reduce_max_precise_f64x2(self, a: f64x2<Self>) -> f64 {
6200        crate::kernel!(
6201            #[inline(always)]
6202            fn kernel(token: Sse2, a: f64x2<Sse2>) -> f64 {
6203                let reduced: __m128d = a.into();
6204                let shifted = _mm_castsi128_pd(_mm_srli_si128::<8>(_mm_castpd_si128(reduced)));
6205                let reduced = {
6206                    let intermediate = _mm_max_pd(reduced, shifted);
6207                    let shifted_is_nan = _mm_cmpunord_pd(shifted, shifted);
6208                    _mm_or_pd(
6209                        _mm_and_pd(shifted_is_nan, reduced),
6210                        _mm_andnot_pd(shifted_is_nan, intermediate),
6211                    )
6212                };
6213                _mm_cvtsd_f64(reduced)
6214            }
6215        );
6216        kernel(self, a)
6217    }
6218    #[inline(always)]
6219    fn reduce_min_precise_f64x2(self, a: f64x2<Self>) -> f64 {
6220        crate::kernel!(
6221            #[inline(always)]
6222            fn kernel(token: Sse2, a: f64x2<Sse2>) -> f64 {
6223                let reduced: __m128d = a.into();
6224                let shifted = _mm_castsi128_pd(_mm_srli_si128::<8>(_mm_castpd_si128(reduced)));
6225                let reduced = {
6226                    let intermediate = _mm_min_pd(reduced, shifted);
6227                    let shifted_is_nan = _mm_cmpunord_pd(shifted, shifted);
6228                    _mm_or_pd(
6229                        _mm_and_pd(shifted_is_nan, reduced),
6230                        _mm_andnot_pd(shifted_is_nan, intermediate),
6231                    )
6232                };
6233                _mm_cvtsd_f64(reduced)
6234            }
6235        );
6236        kernel(self, a)
6237    }
6238    #[inline(always)]
6239    fn reduce_sum_f64x2(self, a: f64x2<Self>) -> f64 {
6240        crate::kernel!(
6241            #[inline(always)]
6242            fn kernel(token: Sse2, a: f64x2<Sse2>) -> f64 {
6243                let a: __m128d = a.into();
6244                _mm_cvtsd_f64(_mm_add_sd(a, _mm_unpackhi_pd(a, a)))
6245            }
6246        );
6247        kernel(self, a)
6248    }
6249    #[inline(always)]
6250    fn reduce_product_f64x2(self, a: f64x2<Self>) -> f64 {
6251        crate::kernel!(
6252            #[inline(always)]
6253            fn kernel(token: Sse2, a: f64x2<Sse2>) -> f64 {
6254                let a: __m128d = a.into();
6255                _mm_cvtsd_f64(_mm_mul_sd(a, _mm_unpackhi_pd(a, a)))
6256            }
6257        );
6258        kernel(self, a)
6259    }
6260    #[inline(always)]
6261    fn max_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6262        crate::kernel!(
6263            #[inline(always)]
6264            fn kernel(token: Sse2, a: f64x2<Sse2>, b: f64x2<Sse2>) -> f64x2<Sse2> {
6265                _mm_max_pd(a.into(), b.into()).simd_into(token)
6266            }
6267        );
6268        kernel(self, a, b)
6269    }
6270    #[inline(always)]
6271    fn min_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6272        crate::kernel!(
6273            #[inline(always)]
6274            fn kernel(token: Sse2, a: f64x2<Sse2>, b: f64x2<Sse2>) -> f64x2<Sse2> {
6275                _mm_min_pd(a.into(), b.into()).simd_into(token)
6276            }
6277        );
6278        kernel(self, a, b)
6279    }
6280    #[inline(always)]
6281    fn max_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6282        crate::kernel!(
6283            #[inline(always)]
6284            fn kernel(token: Sse2, a: f64x2<Sse2>, b: f64x2<Sse2>) -> f64x2<Sse2> {
6285                let a = a.into();
6286                let b = b.into();
6287                let intermediate = _mm_max_pd(a, b);
6288                let b_is_nan = _mm_cmpunord_pd(b, b);
6289                _mm_or_pd(
6290                    _mm_and_pd(b_is_nan, a),
6291                    _mm_andnot_pd(b_is_nan, intermediate),
6292                )
6293                .simd_into(token)
6294            }
6295        );
6296        kernel(self, a, b)
6297    }
6298    #[inline(always)]
6299    fn min_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6300        crate::kernel!(
6301            #[inline(always)]
6302            fn kernel(token: Sse2, a: f64x2<Sse2>, b: f64x2<Sse2>) -> f64x2<Sse2> {
6303                let a = a.into();
6304                let b = b.into();
6305                let intermediate = _mm_min_pd(a, b);
6306                let b_is_nan = _mm_cmpunord_pd(b, b);
6307                _mm_or_pd(
6308                    _mm_and_pd(b_is_nan, a),
6309                    _mm_andnot_pd(b_is_nan, intermediate),
6310                )
6311                .simd_into(token)
6312            }
6313        );
6314        kernel(self, a, b)
6315    }
6316    #[inline(always)]
6317    fn clamp_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
6318        crate::kernel!(
6319            #[inline(always)]
6320            fn kernel(token: Sse2, a: f64x2<Sse2>, b: f64x2<Sse2>, c: f64x2<Sse2>) -> f64x2<Sse2> {
6321                debug_assert!(
6322                    b.simd_le(c).all_true(),
6323                    "clamp requires min <= max for all elements"
6324                );
6325                _mm_min_pd(c.into(), _mm_max_pd(b.into(), a.into())).simd_into(token)
6326            }
6327        );
6328        kernel(self, a, b, c)
6329    }
6330    #[inline(always)]
6331    fn simd_eq_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
6332        crate::kernel!(
6333            #[inline(always)]
6334            fn kernel(token: Sse2, a: f64x2<Sse2>, b: f64x2<Sse2>) -> mask64x2<Sse2> {
6335                _mm_castpd_si128(_mm_cmpeq_pd(a.into(), b.into())).simd_into(token)
6336            }
6337        );
6338        kernel(self, a, b)
6339    }
6340    #[inline(always)]
6341    fn simd_ne_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
6342        crate::kernel!(
6343            #[inline(always)]
6344            fn kernel(token: Sse2, a: f64x2<Sse2>, b: f64x2<Sse2>) -> mask64x2<Sse2> {
6345                _mm_castpd_si128(_mm_cmpneq_pd(a.into(), b.into())).simd_into(token)
6346            }
6347        );
6348        kernel(self, a, b)
6349    }
6350    #[inline(always)]
6351    fn simd_lt_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
6352        crate::kernel!(
6353            #[inline(always)]
6354            fn kernel(token: Sse2, a: f64x2<Sse2>, b: f64x2<Sse2>) -> mask64x2<Sse2> {
6355                _mm_castpd_si128(_mm_cmplt_pd(a.into(), b.into())).simd_into(token)
6356            }
6357        );
6358        kernel(self, a, b)
6359    }
6360    #[inline(always)]
6361    fn simd_le_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
6362        crate::kernel!(
6363            #[inline(always)]
6364            fn kernel(token: Sse2, a: f64x2<Sse2>, b: f64x2<Sse2>) -> mask64x2<Sse2> {
6365                _mm_castpd_si128(_mm_cmple_pd(a.into(), b.into())).simd_into(token)
6366            }
6367        );
6368        kernel(self, a, b)
6369    }
6370    #[inline(always)]
6371    fn zip_low_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6372        crate::kernel!(
6373            #[inline(always)]
6374            fn kernel(token: Sse2, a: f64x2<Sse2>, b: f64x2<Sse2>) -> f64x2<Sse2> {
6375                _mm_unpacklo_pd(a.into(), b.into()).simd_into(token)
6376            }
6377        );
6378        kernel(self, a, b)
6379    }
6380    #[inline(always)]
6381    fn zip_high_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6382        crate::kernel!(
6383            #[inline(always)]
6384            fn kernel(token: Sse2, a: f64x2<Sse2>, b: f64x2<Sse2>) -> f64x2<Sse2> {
6385                _mm_unpackhi_pd(a.into(), b.into()).simd_into(token)
6386            }
6387        );
6388        kernel(self, a, b)
6389    }
6390    #[inline(always)]
6391    fn unzip_low_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6392        crate::kernel!(
6393            #[inline(always)]
6394            fn kernel(token: Sse2, a: f64x2<Sse2>, b: f64x2<Sse2>) -> f64x2<Sse2> {
6395                _mm_shuffle_pd::<0b00>(a.into(), b.into()).simd_into(token)
6396            }
6397        );
6398        kernel(self, a, b)
6399    }
6400    #[inline(always)]
6401    fn unzip_high_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
6402        crate::kernel!(
6403            #[inline(always)]
6404            fn kernel(token: Sse2, a: f64x2<Sse2>, b: f64x2<Sse2>) -> f64x2<Sse2> {
6405                _mm_shuffle_pd::<0b11>(a.into(), b.into()).simd_into(token)
6406            }
6407        );
6408        kernel(self, a, b)
6409    }
6410    #[inline(always)]
6411    fn interleave_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> (f64x2<Self>, f64x2<Self>) {
6412        (self.zip_low_f64x2(a, b), self.zip_high_f64x2(a, b))
6413    }
6414    #[inline(always)]
6415    fn deinterleave_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> (f64x2<Self>, f64x2<Self>) {
6416        (self.unzip_low_f64x2(a, b), self.unzip_high_f64x2(a, b))
6417    }
6418    #[inline(always)]
6419    fn mul_add_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
6420        a * b + c
6421    }
6422    #[inline(always)]
6423    fn mul_add_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
6424        crate::math::mul_add_precise_f64x2(self, a, b, c)
6425    }
6426    #[inline(always)]
6427    fn mul_sub_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
6428        a * b - c
6429    }
6430    #[inline(always)]
6431    fn mul_sub_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
6432        self.mul_add_precise_f64x2(a, b, -c)
6433    }
6434    #[inline(always)]
6435    fn floor_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6436        [f64::floor(a[0usize]), f64::floor(a[1usize])].simd_into(self)
6437    }
6438    #[inline(always)]
6439    fn ceil_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6440        [f64::ceil(a[0usize]), f64::ceil(a[1usize])].simd_into(self)
6441    }
6442    #[inline(always)]
6443    fn round_ties_even_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6444        [
6445            f64::round_ties_even(a[0usize]),
6446            f64::round_ties_even(a[1usize]),
6447        ]
6448        .simd_into(self)
6449    }
6450    #[inline(always)]
6451    fn round_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6452        let bias = self.splat_f64x2(const { f64::next_down(0.5) });
6453        (a + self.copysign_f64x2(bias, a)).trunc()
6454    }
6455    #[inline(always)]
6456    fn fract_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6457        a - self.trunc_f64x2(a)
6458    }
6459    #[inline(always)]
6460    fn trunc_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6461        [f64::trunc(a[0usize]), f64::trunc(a[1usize])].simd_into(self)
6462    }
6463    #[inline(always)]
6464    fn to_degrees_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6465        a * const { f64::to_degrees(1.0) }
6466    }
6467    #[inline(always)]
6468    fn to_radians_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
6469        a * const { f64::to_radians(1.0) }
6470    }
6471    #[inline(always)]
6472    fn is_nan_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6473        a.simd_ne(a)
6474    }
6475    #[inline(always)]
6476    fn is_infinite_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6477        a.abs().simd_eq(f64::INFINITY)
6478    }
6479    #[inline(always)]
6480    fn is_finite_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6481        a.abs().simd_lt(f64::INFINITY)
6482    }
6483    #[inline(always)]
6484    fn is_subnormal_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6485        let i: i64x2<Self> = a.bitcast();
6486        (i & 0x7FF0_0000_0000_0000_i64).simd_eq(0) & (i & 0x000F_FFFF_FFFF_FFFF_i64).simd_ne(0)
6487    }
6488    #[inline(always)]
6489    fn is_normal_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6490        let i: i64x2<Self> = a.bitcast();
6491        let exp = i & 0x7FF0_0000_0000_0000_i64;
6492        exp.simd_ne(0) & exp.simd_ne(0x7FF0_0000_0000_0000_i64)
6493    }
6494    #[inline(always)]
6495    fn is_sign_positive_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6496        let i: i64x2<Self> = a.bitcast();
6497        (i & i64::MIN).simd_eq(0)
6498    }
6499    #[inline(always)]
6500    fn is_sign_negative_f64x2(self, a: f64x2<Self>) -> mask64x2<Self> {
6501        !a.is_sign_positive()
6502    }
6503    #[inline(always)]
6504    fn select_f64x2(self, a: mask64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
6505        crate::kernel!(
6506            #[inline(always)]
6507            fn kernel(
6508                token: Sse2,
6509                a: mask64x2<Sse2>,
6510                b: f64x2<Sse2>,
6511                c: f64x2<Sse2>,
6512            ) -> f64x2<Sse2> {
6513                _mm_or_pd(
6514                    _mm_and_pd(_mm_castsi128_pd(a.into()), b.into()),
6515                    _mm_andnot_pd(_mm_castsi128_pd(a.into()), c.into()),
6516                )
6517                .simd_into(token)
6518            }
6519        );
6520        kernel(self, a, b, c)
6521    }
6522    #[inline(always)]
6523    fn combine_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x4<Self> {
6524        f64x4 {
6525            val: crate::support::Aligned256([a.val.0, b.val.0]),
6526            simd: self,
6527        }
6528    }
6529    #[inline(always)]
6530    fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2<Self>; 4usize] {
6531        crate::kernel!(
6532            #[inline(always)]
6533            fn kernel(token: Sse2, src: &[f64; 8usize]) -> [f64x2<Sse2>; 4usize] {
6534                let (chunks, []) = src.as_chunks::<2usize>() else {
6535                    unreachable!()
6536                };
6537                let v0: __m128d =
6538                    crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[0]);
6539                let v1: __m128d =
6540                    crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[1]);
6541                let v2: __m128d =
6542                    crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[2]);
6543                let v3: __m128d =
6544                    crate::transmute::checked_transmute_copy::<[f64; 2usize], __m128d>(&chunks[3]);
6545                let out0 = _mm_unpacklo_pd(v0, v2);
6546                let out1 = _mm_unpackhi_pd(v0, v2);
6547                let out2 = _mm_unpacklo_pd(v1, v3);
6548                let out3 = _mm_unpackhi_pd(v1, v3);
6549                [
6550                    out0.simd_into(token),
6551                    out1.simd_into(token),
6552                    out2.simd_into(token),
6553                    out3.simd_into(token),
6554                ]
6555            }
6556        );
6557        kernel(self, src)
6558    }
6559    #[inline(always)]
6560    fn store_four_interleaved_f64x2(
6561        self,
6562        vectors: [f64x2<Self>; 4usize],
6563        dest: &mut [f64; 8usize],
6564    ) -> () {
6565        crate::kernel!(
6566            #[inline(always)]
6567            fn kernel(token: Sse2, vectors: [f64x2<Sse2>; 4usize], dest: &mut [f64; 8usize]) -> () {
6568                let _ = token;
6569                let v0: __m128d = vectors[0].into();
6570                let v1: __m128d = vectors[1].into();
6571                let v2: __m128d = vectors[2].into();
6572                let v3: __m128d = vectors[3].into();
6573                let out0 = _mm_unpacklo_pd(v0, v1);
6574                let out1 = _mm_unpacklo_pd(v2, v3);
6575                let out2 = _mm_unpackhi_pd(v0, v1);
6576                let out3 = _mm_unpackhi_pd(v2, v3);
6577                let (chunks, []) = dest.as_chunks_mut::<2usize>() else {
6578                    unreachable!()
6579                };
6580                crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>(
6581                    out0,
6582                    &mut chunks[0],
6583                );
6584                crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>(
6585                    out1,
6586                    &mut chunks[1],
6587                );
6588                crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>(
6589                    out2,
6590                    &mut chunks[2],
6591                );
6592                crate::transmute::checked_transmute_store::<__m128d, [f64; 2usize]>(
6593                    out3,
6594                    &mut chunks[3],
6595                );
6596            }
6597        );
6598        kernel(self, vectors, dest);
6599    }
6600    #[inline(always)]
6601    fn narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
6602        crate::kernel!(
6603            #[inline(always)]
6604            fn kernel(token: Sse2, a: f64x2<Sse2>, b: f64x2<Sse2>) -> f32x4<Sse2> {
6605                let low = _mm_cvtpd_ps(a.into());
6606                let high = _mm_cvtpd_ps(b.into());
6607                _mm_movelh_ps(low, high).simd_into(token)
6608            }
6609        );
6610        kernel(self, a, b)
6611    }
6612    #[inline(always)]
6613    fn saturating_narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
6614        self.narrow_f64x2(a, b)
6615    }
6616    #[inline(always)]
6617    fn relaxed_narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
6618        self.narrow_f64x2(a, b)
6619    }
6620    #[inline(always)]
6621    fn cvt_u64_f64x2(self, a: f64x2<Self>) -> u64x2<Self> {
6622        [a[0usize] as u64, a[1usize] as u64].simd_into(self)
6623    }
6624    #[inline(always)]
6625    fn cvt_u64_precise_f64x2(self, a: f64x2<Self>) -> u64x2<Self> {
6626        [a[0usize] as u64, a[1usize] as u64].simd_into(self)
6627    }
6628    #[inline(always)]
6629    #[cfg(target_arch = "x86_64")]
6630    fn cvt_i64_f64x2(self, a: f64x2<Self>) -> i64x2<Self> {
6631        crate::kernel!(
6632            #[inline(always)]
6633            fn kernel(token: Sse2, a: f64x2<Sse2>) -> i64x2<Sse2> {
6634                let a = a.into();
6635                let low = _mm_cvttsd_si64(a);
6636                let high = _mm_cvttsd_si64(_mm_unpackhi_pd(a, a));
6637                _mm_set_epi64x(high, low).simd_into(token)
6638            }
6639        );
6640        kernel(self, a)
6641    }
6642    #[cfg(target_arch = "x86")]
6643    fn cvt_i64_f64x2(self, a: f64x2<Self>) -> i64x2<Self> {
6644        [a[0usize] as i64, a[1usize] as i64].simd_into(self)
6645    }
6646    #[inline(always)]
6647    fn cvt_i64_precise_f64x2(self, a: f64x2<Self>) -> i64x2<Self> {
6648        [a[0usize] as i64, a[1usize] as i64].simd_into(self)
6649    }
6650    #[inline(always)]
6651    fn abs_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6652        [i64::wrapping_abs(a[0usize]), i64::wrapping_abs(a[1usize])].simd_into(self)
6653    }
6654    #[inline(always)]
6655    fn splat_i64x2(self, val: i64) -> i64x2<Self> {
6656        crate::kernel!(
6657            #[inline(always)]
6658            fn kernel(token: Sse2, val: i64) -> i64x2<Sse2> {
6659                _mm_set1_epi64x(val).simd_into(token)
6660            }
6661        );
6662        kernel(self, val)
6663    }
6664    #[inline(always)]
6665    fn reverse_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6666        [a[1usize], a[0usize]].simd_into(self)
6667    }
6668    #[inline(always)]
6669    fn slide_i64x2<const SHIFT: usize>(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6670        if SHIFT >= 2usize {
6671            return b;
6672        }
6673        let result = dyn_alignr_128(
6674            self,
6675            Bytes::to_bytes(b).val.0,
6676            Bytes::to_bytes(a).val.0,
6677            SHIFT * 8usize,
6678        );
6679        Bytes::from_bytes(u8x16 {
6680            val: crate::support::Aligned128(result),
6681            simd: self,
6682        })
6683    }
6684    #[inline(always)]
6685    fn compress_i64x2(self, values: i64x2<Self>, mask: mask64x2<Self>) -> i64x2<Self> {
6686        let mask = mask8x16 {
6687            val: crate::transmute::checked_transmute_copy(&mask.val),
6688            simd: self,
6689        };
6690        Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask))
6691    }
6692    #[inline(always)]
6693    fn compress_merge_i64x2(
6694        self,
6695        values: i64x2<Self>,
6696        mask: mask64x2<Self>,
6697        merge: i64x2<Self>,
6698    ) -> i64x2<Self> {
6699        let mask = mask8x16 {
6700            val: crate::transmute::checked_transmute_copy(&mask.val),
6701            simd: self,
6702        };
6703        Bytes::from_bytes(self.compress_merge_u8x16(
6704            Bytes::to_bytes(values),
6705            mask,
6706            Bytes::to_bytes(merge),
6707        ))
6708    }
6709    #[inline(always)]
6710    fn expand_i64x2(self, values: i64x2<Self>, mask: mask64x2<Self>) -> i64x2<Self> {
6711        let mask = mask8x16 {
6712            val: crate::transmute::checked_transmute_copy(&mask.val),
6713            simd: self,
6714        };
6715        Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask))
6716    }
6717    #[inline(always)]
6718    fn expand_merge_i64x2(
6719        self,
6720        values: i64x2<Self>,
6721        mask: mask64x2<Self>,
6722        merge: i64x2<Self>,
6723    ) -> i64x2<Self> {
6724        let mask = mask8x16 {
6725            val: crate::transmute::checked_transmute_copy(&mask.val),
6726            simd: self,
6727        };
6728        Bytes::from_bytes(self.expand_merge_u8x16(
6729            Bytes::to_bytes(values),
6730            mask,
6731            Bytes::to_bytes(merge),
6732        ))
6733    }
6734    #[inline(always)]
6735    fn count_ones_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6736        [
6737            i64::from(a[0usize].count_ones()),
6738            i64::from(a[1usize].count_ones()),
6739        ]
6740        .simd_into(self)
6741    }
6742    #[inline(always)]
6743    fn count_zeros_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6744        [
6745            i64::from(a[0usize].count_zeros()),
6746            i64::from(a[1usize].count_zeros()),
6747        ]
6748        .simd_into(self)
6749    }
6750    #[inline(always)]
6751    fn add_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6752        crate::kernel!(
6753            #[inline(always)]
6754            fn kernel(token: Sse2, a: i64x2<Sse2>, b: i64x2<Sse2>) -> i64x2<Sse2> {
6755                _mm_add_epi64(a.into(), b.into()).simd_into(token)
6756            }
6757        );
6758        kernel(self, a, b)
6759    }
6760    #[inline(always)]
6761    fn saturating_add_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6762        [
6763            i64::saturating_add(a[0usize], b[0usize]),
6764            i64::saturating_add(a[1usize], b[1usize]),
6765        ]
6766        .simd_into(self)
6767    }
6768    #[inline(always)]
6769    fn sub_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6770        crate::kernel!(
6771            #[inline(always)]
6772            fn kernel(token: Sse2, a: i64x2<Sse2>, b: i64x2<Sse2>) -> i64x2<Sse2> {
6773                _mm_sub_epi64(a.into(), b.into()).simd_into(token)
6774            }
6775        );
6776        kernel(self, a, b)
6777    }
6778    #[inline(always)]
6779    fn saturating_sub_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6780        [
6781            i64::saturating_sub(a[0usize], b[0usize]),
6782            i64::saturating_sub(a[1usize], b[1usize]),
6783        ]
6784        .simd_into(self)
6785    }
6786    #[inline(always)]
6787    fn mul_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6788        crate::kernel!(
6789            #[inline(always)]
6790            fn kernel(token: Sse2, a: i64x2<Sse2>, b: i64x2<Sse2>) -> i64x2<Sse2> {
6791                let a = a.into();
6792                let b = b.into();
6793                let a_high = _mm_shuffle_epi32::<0xf5>(a);
6794                let b_high = _mm_shuffle_epi32::<0xf5>(b);
6795                let cross = _mm_add_epi64(_mm_mul_epu32(a_high, b), _mm_mul_epu32(a, b_high));
6796                let low = _mm_mul_epu32(a, b);
6797                _mm_add_epi64(low, _mm_slli_epi64::<32>(cross)).simd_into(token)
6798            }
6799        );
6800        kernel(self, a, b)
6801    }
6802    #[inline(always)]
6803    fn and_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6804        crate::kernel!(
6805            #[inline(always)]
6806            fn kernel(token: Sse2, a: i64x2<Sse2>, b: i64x2<Sse2>) -> i64x2<Sse2> {
6807                _mm_and_si128(a.into(), b.into()).simd_into(token)
6808            }
6809        );
6810        kernel(self, a, b)
6811    }
6812    #[inline(always)]
6813    fn or_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6814        crate::kernel!(
6815            #[inline(always)]
6816            fn kernel(token: Sse2, a: i64x2<Sse2>, b: i64x2<Sse2>) -> i64x2<Sse2> {
6817                _mm_or_si128(a.into(), b.into()).simd_into(token)
6818            }
6819        );
6820        kernel(self, a, b)
6821    }
6822    #[inline(always)]
6823    fn xor_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6824        crate::kernel!(
6825            #[inline(always)]
6826            fn kernel(token: Sse2, a: i64x2<Sse2>, b: i64x2<Sse2>) -> i64x2<Sse2> {
6827                _mm_xor_si128(a.into(), b.into()).simd_into(token)
6828            }
6829        );
6830        kernel(self, a, b)
6831    }
6832    #[inline(always)]
6833    fn not_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
6834        a ^ !0
6835    }
6836    #[inline(always)]
6837    fn shl_i64x2(self, a: i64x2<Self>, shift: u32) -> i64x2<Self> {
6838        crate::kernel!(
6839            #[inline(always)]
6840            fn kernel(token: Sse2, a: i64x2<Sse2>, shift: u32) -> i64x2<Sse2> {
6841                _mm_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
6842            }
6843        );
6844        kernel(self, a, shift)
6845    }
6846    #[inline(always)]
6847    fn shlv_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6848        [
6849            i64::wrapping_shl(a[0usize], b[0usize] as u32),
6850            i64::wrapping_shl(a[1usize], b[1usize] as u32),
6851        ]
6852        .simd_into(self)
6853    }
6854    #[inline(always)]
6855    fn shr_i64x2(self, a: i64x2<Self>, shift: u32) -> i64x2<Self> {
6856        crate::kernel!(
6857            #[inline(always)]
6858            fn kernel(token: Sse2, a: i64x2<Sse2>, shift: u32) -> i64x2<Sse2> {
6859                let value = a.into();
6860                let count = _mm_cvtsi32_si128(shift.cast_signed());
6861                let shifted_bias = _mm_srl_epi64(_mm_set1_epi64x(i64::MIN), count);
6862                let shifted = _mm_srl_epi64(value, count);
6863                _mm_sub_epi64(_mm_xor_si128(shifted, shifted_bias), shifted_bias).simd_into(token)
6864            }
6865        );
6866        kernel(self, a, shift)
6867    }
6868    #[inline(always)]
6869    fn shrv_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6870        [
6871            i64::wrapping_shr(a[0usize], b[0usize] as u32),
6872            i64::wrapping_shr(a[1usize], b[1usize] as u32),
6873        ]
6874        .simd_into(self)
6875    }
6876    #[inline(always)]
6877    fn reduce_max_i64x2(self, a: i64x2<Self>) -> i64 {
6878        crate::kernel!(
6879            #[inline(always)]
6880            fn kernel(token: Sse2, a: i64x2<Sse2>) -> i64 {
6881                let lanes: [i64; 2] = a.into();
6882                lanes[0].max(lanes[1])
6883            }
6884        );
6885        kernel(self, a)
6886    }
6887    #[inline(always)]
6888    fn reduce_min_i64x2(self, a: i64x2<Self>) -> i64 {
6889        crate::kernel!(
6890            #[inline(always)]
6891            fn kernel(token: Sse2, a: i64x2<Sse2>) -> i64 {
6892                let lanes: [i64; 2] = a.into();
6893                lanes[0].min(lanes[1])
6894            }
6895        );
6896        kernel(self, a)
6897    }
6898    #[inline(always)]
6899    fn reduce_sum_i64x2(self, a: i64x2<Self>) -> i64 {
6900        crate::kernel!(
6901            #[inline(always)]
6902            fn kernel(token: Sse2, a: i64x2<Sse2>) -> i64 {
6903                let sum: __m128i = a.into();
6904                let sum = _mm_add_epi64(sum, _mm_srli_si128::<8>(sum));
6905                let lanes: [i64; 2usize] = crate::transmute::checked_transmute_copy(&sum);
6906                lanes[0]
6907            }
6908        );
6909        kernel(self, a)
6910    }
6911    #[inline(always)]
6912    fn reduce_product_i64x2(self, a: i64x2<Self>) -> i64 {
6913        let mul_level_0: [i64; 1usize] = [a[0usize].wrapping_mul(a[1usize])];
6914        mul_level_0[0]
6915    }
6916    #[inline(always)]
6917    fn max_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6918        [
6919            i64::max(a[0usize], b[0usize]),
6920            i64::max(a[1usize], b[1usize]),
6921        ]
6922        .simd_into(self)
6923    }
6924    #[inline(always)]
6925    fn min_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6926        [
6927            i64::min(a[0usize], b[0usize]),
6928            i64::min(a[1usize], b[1usize]),
6929        ]
6930        .simd_into(self)
6931    }
6932    #[inline(always)]
6933    fn clamp_i64x2(self, a: i64x2<Self>, b: i64x2<Self>, c: i64x2<Self>) -> i64x2<Self> {
6934        debug_assert!(
6935            b.simd_le(c).all_true(),
6936            "clamp requires min <= max for all elements"
6937        );
6938        a.max(b).min(c)
6939    }
6940    #[inline(always)]
6941    fn simd_eq_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
6942        crate::kernel!(
6943            #[inline(always)]
6944            fn kernel(token: Sse2, a: i64x2<Sse2>, b: i64x2<Sse2>) -> mask64x2<Sse2> {
6945                {
6946                    let eq32 = _mm_cmpeq_epi32(a.into(), b.into());
6947                    let swapped = _mm_shuffle_epi32::<0b10_11_00_01>(eq32);
6948                    _mm_and_si128(eq32, swapped)
6949                }
6950                .simd_into(token)
6951            }
6952        );
6953        kernel(self, a, b)
6954    }
6955    #[inline(always)]
6956    fn simd_lt_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
6957        [
6958            -(i64::lt(&a[0usize], &b[0usize]) as i64),
6959            -(i64::lt(&a[1usize], &b[1usize]) as i64),
6960        ]
6961        .simd_into(self)
6962    }
6963    #[inline(always)]
6964    fn simd_le_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
6965        [
6966            -(i64::le(&a[0usize], &b[0usize]) as i64),
6967            -(i64::le(&a[1usize], &b[1usize]) as i64),
6968        ]
6969        .simd_into(self)
6970    }
6971    #[inline(always)]
6972    fn zip_low_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6973        crate::kernel!(
6974            #[inline(always)]
6975            fn kernel(token: Sse2, a: i64x2<Sse2>, b: i64x2<Sse2>) -> i64x2<Sse2> {
6976                _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token)
6977            }
6978        );
6979        kernel(self, a, b)
6980    }
6981    #[inline(always)]
6982    fn zip_high_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6983        crate::kernel!(
6984            #[inline(always)]
6985            fn kernel(token: Sse2, a: i64x2<Sse2>, b: i64x2<Sse2>) -> i64x2<Sse2> {
6986                _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token)
6987            }
6988        );
6989        kernel(self, a, b)
6990    }
6991    #[inline(always)]
6992    fn unzip_low_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
6993        crate::kernel!(
6994            #[inline(always)]
6995            fn kernel(token: Sse2, a: i64x2<Sse2>, b: i64x2<Sse2>) -> i64x2<Sse2> {
6996                _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token)
6997            }
6998        );
6999        kernel(self, a, b)
7000    }
7001    #[inline(always)]
7002    fn unzip_high_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
7003        crate::kernel!(
7004            #[inline(always)]
7005            fn kernel(token: Sse2, a: i64x2<Sse2>, b: i64x2<Sse2>) -> i64x2<Sse2> {
7006                _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token)
7007            }
7008        );
7009        kernel(self, a, b)
7010    }
7011    #[inline(always)]
7012    fn interleave_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> (i64x2<Self>, i64x2<Self>) {
7013        (self.zip_low_i64x2(a, b), self.zip_high_i64x2(a, b))
7014    }
7015    #[inline(always)]
7016    fn deinterleave_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> (i64x2<Self>, i64x2<Self>) {
7017        (self.unzip_low_i64x2(a, b), self.unzip_high_i64x2(a, b))
7018    }
7019    #[inline(always)]
7020    fn select_i64x2(self, a: mask64x2<Self>, b: i64x2<Self>, c: i64x2<Self>) -> i64x2<Self> {
7021        crate::kernel!(
7022            #[inline(always)]
7023            fn kernel(
7024                token: Sse2,
7025                a: mask64x2<Sse2>,
7026                b: i64x2<Sse2>,
7027                c: i64x2<Sse2>,
7028            ) -> i64x2<Sse2> {
7029                _mm_or_si128(
7030                    _mm_and_si128(a.into(), b.into()),
7031                    _mm_andnot_si128(a.into(), c.into()),
7032                )
7033                .simd_into(token)
7034            }
7035        );
7036        kernel(self, a, b, c)
7037    }
7038    #[inline(always)]
7039    fn combine_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x4<Self> {
7040        i64x4 {
7041            val: crate::support::Aligned256([a.val.0, b.val.0]),
7042            simd: self,
7043        }
7044    }
7045    #[inline(always)]
7046    fn neg_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
7047        crate::kernel!(
7048            #[inline(always)]
7049            fn kernel(token: Sse2, a: i64x2<Sse2>) -> i64x2<Sse2> {
7050                _mm_sub_epi64(_mm_setzero_si128(), a.into()).simd_into(token)
7051            }
7052        );
7053        kernel(self, a)
7054    }
7055    #[inline(always)]
7056    fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2<Self>; 4usize] {
7057        crate::kernel!(
7058            #[inline(always)]
7059            fn kernel(token: Sse2, src: &[i64; 8usize]) -> [i64x2<Sse2>; 4usize] {
7060                let (chunks, []) = src.as_chunks::<2usize>() else {
7061                    unreachable!()
7062                };
7063                let v0: __m128i =
7064                    crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[0]);
7065                let v1: __m128i =
7066                    crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[1]);
7067                let v2: __m128i =
7068                    crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[2]);
7069                let v3: __m128i =
7070                    crate::transmute::checked_transmute_copy::<[i64; 2usize], __m128i>(&chunks[3]);
7071                let out0 = _mm_unpacklo_epi64(v0, v2);
7072                let out1 = _mm_unpackhi_epi64(v0, v2);
7073                let out2 = _mm_unpacklo_epi64(v1, v3);
7074                let out3 = _mm_unpackhi_epi64(v1, v3);
7075                [
7076                    out0.simd_into(token),
7077                    out1.simd_into(token),
7078                    out2.simd_into(token),
7079                    out3.simd_into(token),
7080                ]
7081            }
7082        );
7083        kernel(self, src)
7084    }
7085    #[inline(always)]
7086    fn store_four_interleaved_i64x2(
7087        self,
7088        vectors: [i64x2<Self>; 4usize],
7089        dest: &mut [i64; 8usize],
7090    ) -> () {
7091        crate::kernel!(
7092            #[inline(always)]
7093            fn kernel(token: Sse2, vectors: [i64x2<Sse2>; 4usize], dest: &mut [i64; 8usize]) -> () {
7094                let _ = token;
7095                let v0: __m128i = vectors[0].into();
7096                let v1: __m128i = vectors[1].into();
7097                let v2: __m128i = vectors[2].into();
7098                let v3: __m128i = vectors[3].into();
7099                let out0 = _mm_unpacklo_epi64(v0, v1);
7100                let out1 = _mm_unpacklo_epi64(v2, v3);
7101                let out2 = _mm_unpackhi_epi64(v0, v1);
7102                let out3 = _mm_unpackhi_epi64(v2, v3);
7103                let (chunks, []) = dest.as_chunks_mut::<2usize>() else {
7104                    unreachable!()
7105                };
7106                crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>(
7107                    out0,
7108                    &mut chunks[0],
7109                );
7110                crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>(
7111                    out1,
7112                    &mut chunks[1],
7113                );
7114                crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>(
7115                    out2,
7116                    &mut chunks[2],
7117                );
7118                crate::transmute::checked_transmute_store::<__m128i, [i64; 2usize]>(
7119                    out3,
7120                    &mut chunks[3],
7121                );
7122            }
7123        );
7124        kernel(self, vectors, dest);
7125    }
7126    #[inline(always)]
7127    fn narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
7128        crate::kernel!(
7129            #[inline(always)]
7130            fn kernel(token: Sse2, a: i64x2<Sse2>, b: i64x2<Sse2>) -> i32x4<Sse2> {
7131                _mm_castps_si128(_mm_shuffle_ps::<0x88>(
7132                    _mm_castsi128_ps(a.into()),
7133                    _mm_castsi128_ps(b.into()),
7134                ))
7135                .simd_into(token)
7136            }
7137        );
7138        kernel(self, a, b)
7139    }
7140    #[inline(always)]
7141    fn saturating_narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
7142        [
7143            a[0usize].clamp(i32::MIN as i64, i32::MAX as i64) as i32,
7144            a[1usize].clamp(i32::MIN as i64, i32::MAX as i64) as i32,
7145            b[0usize].clamp(i32::MIN as i64, i32::MAX as i64) as i32,
7146            b[1usize].clamp(i32::MIN as i64, i32::MAX as i64) as i32,
7147        ]
7148        .simd_into(self)
7149    }
7150    #[inline(always)]
7151    fn relaxed_narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
7152        debug_assert!(
7153            a.as_slice()
7154                .iter()
7155                .chain(b.as_slice())
7156                .all(|&value| { value >= i32::MIN as i64 && value <= i32::MAX as i64 }),
7157            "relaxed_narrow inputs must fit in the destination type",
7158        );
7159        self.narrow_i64x2(a, b)
7160    }
7161    #[inline(always)]
7162    fn cvt_f64_i64x2(self, a: i64x2<Self>) -> f64x2<Self> {
7163        [a[0usize] as f64, a[1usize] as f64].simd_into(self)
7164    }
7165    #[inline(always)]
7166    fn splat_u64x2(self, val: u64) -> u64x2<Self> {
7167        crate::kernel!(
7168            #[inline(always)]
7169            fn kernel(token: Sse2, val: u64) -> u64x2<Sse2> {
7170                _mm_set1_epi64x(val.cast_signed()).simd_into(token)
7171            }
7172        );
7173        kernel(self, val)
7174    }
7175    #[inline(always)]
7176    fn reverse_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
7177        [a[1usize], a[0usize]].simd_into(self)
7178    }
7179    #[inline(always)]
7180    fn slide_u64x2<const SHIFT: usize>(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7181        if SHIFT >= 2usize {
7182            return b;
7183        }
7184        let result = dyn_alignr_128(
7185            self,
7186            Bytes::to_bytes(b).val.0,
7187            Bytes::to_bytes(a).val.0,
7188            SHIFT * 8usize,
7189        );
7190        Bytes::from_bytes(u8x16 {
7191            val: crate::support::Aligned128(result),
7192            simd: self,
7193        })
7194    }
7195    #[inline(always)]
7196    fn compress_u64x2(self, values: u64x2<Self>, mask: mask64x2<Self>) -> u64x2<Self> {
7197        let mask = mask8x16 {
7198            val: crate::transmute::checked_transmute_copy(&mask.val),
7199            simd: self,
7200        };
7201        Bytes::from_bytes(self.compress_u8x16(Bytes::to_bytes(values), mask))
7202    }
7203    #[inline(always)]
7204    fn compress_merge_u64x2(
7205        self,
7206        values: u64x2<Self>,
7207        mask: mask64x2<Self>,
7208        merge: u64x2<Self>,
7209    ) -> u64x2<Self> {
7210        let mask = mask8x16 {
7211            val: crate::transmute::checked_transmute_copy(&mask.val),
7212            simd: self,
7213        };
7214        Bytes::from_bytes(self.compress_merge_u8x16(
7215            Bytes::to_bytes(values),
7216            mask,
7217            Bytes::to_bytes(merge),
7218        ))
7219    }
7220    #[inline(always)]
7221    fn expand_u64x2(self, values: u64x2<Self>, mask: mask64x2<Self>) -> u64x2<Self> {
7222        let mask = mask8x16 {
7223            val: crate::transmute::checked_transmute_copy(&mask.val),
7224            simd: self,
7225        };
7226        Bytes::from_bytes(self.expand_u8x16(Bytes::to_bytes(values), mask))
7227    }
7228    #[inline(always)]
7229    fn expand_merge_u64x2(
7230        self,
7231        values: u64x2<Self>,
7232        mask: mask64x2<Self>,
7233        merge: u64x2<Self>,
7234    ) -> u64x2<Self> {
7235        let mask = mask8x16 {
7236            val: crate::transmute::checked_transmute_copy(&mask.val),
7237            simd: self,
7238        };
7239        Bytes::from_bytes(self.expand_merge_u8x16(
7240            Bytes::to_bytes(values),
7241            mask,
7242            Bytes::to_bytes(merge),
7243        ))
7244    }
7245    #[inline(always)]
7246    fn count_ones_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
7247        [
7248            u64::from(a[0usize].count_ones()),
7249            u64::from(a[1usize].count_ones()),
7250        ]
7251        .simd_into(self)
7252    }
7253    #[inline(always)]
7254    fn count_zeros_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
7255        [
7256            u64::from(a[0usize].count_zeros()),
7257            u64::from(a[1usize].count_zeros()),
7258        ]
7259        .simd_into(self)
7260    }
7261    #[inline(always)]
7262    fn add_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7263        crate::kernel!(
7264            #[inline(always)]
7265            fn kernel(token: Sse2, a: u64x2<Sse2>, b: u64x2<Sse2>) -> u64x2<Sse2> {
7266                _mm_add_epi64(a.into(), b.into()).simd_into(token)
7267            }
7268        );
7269        kernel(self, a, b)
7270    }
7271    #[inline(always)]
7272    fn saturating_add_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7273        [
7274            u64::saturating_add(a[0usize], b[0usize]),
7275            u64::saturating_add(a[1usize], b[1usize]),
7276        ]
7277        .simd_into(self)
7278    }
7279    #[inline(always)]
7280    fn sub_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7281        crate::kernel!(
7282            #[inline(always)]
7283            fn kernel(token: Sse2, a: u64x2<Sse2>, b: u64x2<Sse2>) -> u64x2<Sse2> {
7284                _mm_sub_epi64(a.into(), b.into()).simd_into(token)
7285            }
7286        );
7287        kernel(self, a, b)
7288    }
7289    #[inline(always)]
7290    fn saturating_sub_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7291        [
7292            u64::saturating_sub(a[0usize], b[0usize]),
7293            u64::saturating_sub(a[1usize], b[1usize]),
7294        ]
7295        .simd_into(self)
7296    }
7297    #[inline(always)]
7298    fn mul_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7299        crate::kernel!(
7300            #[inline(always)]
7301            fn kernel(token: Sse2, a: u64x2<Sse2>, b: u64x2<Sse2>) -> u64x2<Sse2> {
7302                let a = a.into();
7303                let b = b.into();
7304                let a_high = _mm_shuffle_epi32::<0xf5>(a);
7305                let b_high = _mm_shuffle_epi32::<0xf5>(b);
7306                let cross = _mm_add_epi64(_mm_mul_epu32(a_high, b), _mm_mul_epu32(a, b_high));
7307                let low = _mm_mul_epu32(a, b);
7308                _mm_add_epi64(low, _mm_slli_epi64::<32>(cross)).simd_into(token)
7309            }
7310        );
7311        kernel(self, a, b)
7312    }
7313    #[inline(always)]
7314    fn and_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7315        crate::kernel!(
7316            #[inline(always)]
7317            fn kernel(token: Sse2, a: u64x2<Sse2>, b: u64x2<Sse2>) -> u64x2<Sse2> {
7318                _mm_and_si128(a.into(), b.into()).simd_into(token)
7319            }
7320        );
7321        kernel(self, a, b)
7322    }
7323    #[inline(always)]
7324    fn or_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7325        crate::kernel!(
7326            #[inline(always)]
7327            fn kernel(token: Sse2, a: u64x2<Sse2>, b: u64x2<Sse2>) -> u64x2<Sse2> {
7328                _mm_or_si128(a.into(), b.into()).simd_into(token)
7329            }
7330        );
7331        kernel(self, a, b)
7332    }
7333    #[inline(always)]
7334    fn xor_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7335        crate::kernel!(
7336            #[inline(always)]
7337            fn kernel(token: Sse2, a: u64x2<Sse2>, b: u64x2<Sse2>) -> u64x2<Sse2> {
7338                _mm_xor_si128(a.into(), b.into()).simd_into(token)
7339            }
7340        );
7341        kernel(self, a, b)
7342    }
7343    #[inline(always)]
7344    fn not_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
7345        a ^ !0
7346    }
7347    #[inline(always)]
7348    fn shl_u64x2(self, a: u64x2<Self>, shift: u32) -> u64x2<Self> {
7349        crate::kernel!(
7350            #[inline(always)]
7351            fn kernel(token: Sse2, a: u64x2<Sse2>, shift: u32) -> u64x2<Sse2> {
7352                _mm_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
7353            }
7354        );
7355        kernel(self, a, shift)
7356    }
7357    #[inline(always)]
7358    fn shlv_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7359        [
7360            u64::wrapping_shl(a[0usize], b[0usize] as u32),
7361            u64::wrapping_shl(a[1usize], b[1usize] as u32),
7362        ]
7363        .simd_into(self)
7364    }
7365    #[inline(always)]
7366    fn shr_u64x2(self, a: u64x2<Self>, shift: u32) -> u64x2<Self> {
7367        crate::kernel!(
7368            #[inline(always)]
7369            fn kernel(token: Sse2, a: u64x2<Sse2>, shift: u32) -> u64x2<Sse2> {
7370                _mm_srl_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
7371            }
7372        );
7373        kernel(self, a, shift)
7374    }
7375    #[inline(always)]
7376    fn shrv_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7377        [
7378            u64::wrapping_shr(a[0usize], b[0usize] as u32),
7379            u64::wrapping_shr(a[1usize], b[1usize] as u32),
7380        ]
7381        .simd_into(self)
7382    }
7383    #[inline(always)]
7384    fn reduce_max_u64x2(self, a: u64x2<Self>) -> u64 {
7385        crate::kernel!(
7386            #[inline(always)]
7387            fn kernel(token: Sse2, a: u64x2<Sse2>) -> u64 {
7388                let lanes: [u64; 2] = a.into();
7389                lanes[0].max(lanes[1])
7390            }
7391        );
7392        kernel(self, a)
7393    }
7394    #[inline(always)]
7395    fn reduce_min_u64x2(self, a: u64x2<Self>) -> u64 {
7396        crate::kernel!(
7397            #[inline(always)]
7398            fn kernel(token: Sse2, a: u64x2<Sse2>) -> u64 {
7399                let lanes: [u64; 2] = a.into();
7400                lanes[0].min(lanes[1])
7401            }
7402        );
7403        kernel(self, a)
7404    }
7405    #[inline(always)]
7406    fn reduce_sum_u64x2(self, a: u64x2<Self>) -> u64 {
7407        crate::kernel!(
7408            #[inline(always)]
7409            fn kernel(token: Sse2, a: u64x2<Sse2>) -> u64 {
7410                let sum: __m128i = a.into();
7411                let sum = _mm_add_epi64(sum, _mm_srli_si128::<8>(sum));
7412                let lanes: [u64; 2usize] = crate::transmute::checked_transmute_copy(&sum);
7413                lanes[0]
7414            }
7415        );
7416        kernel(self, a)
7417    }
7418    #[inline(always)]
7419    fn reduce_product_u64x2(self, a: u64x2<Self>) -> u64 {
7420        let mul_level_0: [u64; 1usize] = [a[0usize].wrapping_mul(a[1usize])];
7421        mul_level_0[0]
7422    }
7423    #[inline(always)]
7424    fn max_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7425        [
7426            u64::max(a[0usize], b[0usize]),
7427            u64::max(a[1usize], b[1usize]),
7428        ]
7429        .simd_into(self)
7430    }
7431    #[inline(always)]
7432    fn min_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7433        [
7434            u64::min(a[0usize], b[0usize]),
7435            u64::min(a[1usize], b[1usize]),
7436        ]
7437        .simd_into(self)
7438    }
7439    #[inline(always)]
7440    fn clamp_u64x2(self, a: u64x2<Self>, b: u64x2<Self>, c: u64x2<Self>) -> u64x2<Self> {
7441        debug_assert!(
7442            b.simd_le(c).all_true(),
7443            "clamp requires min <= max for all elements"
7444        );
7445        a.max(b).min(c)
7446    }
7447    #[inline(always)]
7448    fn simd_eq_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
7449        crate::kernel!(
7450            #[inline(always)]
7451            fn kernel(token: Sse2, a: u64x2<Sse2>, b: u64x2<Sse2>) -> mask64x2<Sse2> {
7452                {
7453                    let eq32 = _mm_cmpeq_epi32(a.into(), b.into());
7454                    let swapped = _mm_shuffle_epi32::<0b10_11_00_01>(eq32);
7455                    _mm_and_si128(eq32, swapped)
7456                }
7457                .simd_into(token)
7458            }
7459        );
7460        kernel(self, a, b)
7461    }
7462    #[inline(always)]
7463    fn simd_lt_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
7464        [
7465            -(u64::lt(&a[0usize], &b[0usize]) as i64),
7466            -(u64::lt(&a[1usize], &b[1usize]) as i64),
7467        ]
7468        .simd_into(self)
7469    }
7470    #[inline(always)]
7471    fn simd_le_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
7472        [
7473            -(u64::le(&a[0usize], &b[0usize]) as i64),
7474            -(u64::le(&a[1usize], &b[1usize]) as i64),
7475        ]
7476        .simd_into(self)
7477    }
7478    #[inline(always)]
7479    fn zip_low_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7480        crate::kernel!(
7481            #[inline(always)]
7482            fn kernel(token: Sse2, a: u64x2<Sse2>, b: u64x2<Sse2>) -> u64x2<Sse2> {
7483                _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token)
7484            }
7485        );
7486        kernel(self, a, b)
7487    }
7488    #[inline(always)]
7489    fn zip_high_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7490        crate::kernel!(
7491            #[inline(always)]
7492            fn kernel(token: Sse2, a: u64x2<Sse2>, b: u64x2<Sse2>) -> u64x2<Sse2> {
7493                _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token)
7494            }
7495        );
7496        kernel(self, a, b)
7497    }
7498    #[inline(always)]
7499    fn unzip_low_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7500        crate::kernel!(
7501            #[inline(always)]
7502            fn kernel(token: Sse2, a: u64x2<Sse2>, b: u64x2<Sse2>) -> u64x2<Sse2> {
7503                _mm_unpacklo_epi64(a.into(), b.into()).simd_into(token)
7504            }
7505        );
7506        kernel(self, a, b)
7507    }
7508    #[inline(always)]
7509    fn unzip_high_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
7510        crate::kernel!(
7511            #[inline(always)]
7512            fn kernel(token: Sse2, a: u64x2<Sse2>, b: u64x2<Sse2>) -> u64x2<Sse2> {
7513                _mm_unpackhi_epi64(a.into(), b.into()).simd_into(token)
7514            }
7515        );
7516        kernel(self, a, b)
7517    }
7518    #[inline(always)]
7519    fn interleave_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> (u64x2<Self>, u64x2<Self>) {
7520        (self.zip_low_u64x2(a, b), self.zip_high_u64x2(a, b))
7521    }
7522    #[inline(always)]
7523    fn deinterleave_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> (u64x2<Self>, u64x2<Self>) {
7524        (self.unzip_low_u64x2(a, b), self.unzip_high_u64x2(a, b))
7525    }
7526    #[inline(always)]
7527    fn select_u64x2(self, a: mask64x2<Self>, b: u64x2<Self>, c: u64x2<Self>) -> u64x2<Self> {
7528        crate::kernel!(
7529            #[inline(always)]
7530            fn kernel(
7531                token: Sse2,
7532                a: mask64x2<Sse2>,
7533                b: u64x2<Sse2>,
7534                c: u64x2<Sse2>,
7535            ) -> u64x2<Sse2> {
7536                _mm_or_si128(
7537                    _mm_and_si128(a.into(), b.into()),
7538                    _mm_andnot_si128(a.into(), c.into()),
7539                )
7540                .simd_into(token)
7541            }
7542        );
7543        kernel(self, a, b, c)
7544    }
7545    #[inline(always)]
7546    fn combine_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x4<Self> {
7547        u64x4 {
7548            val: crate::support::Aligned256([a.val.0, b.val.0]),
7549            simd: self,
7550        }
7551    }
7552    #[inline(always)]
7553    fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2<Self>; 4usize] {
7554        crate::kernel!(
7555            #[inline(always)]
7556            fn kernel(token: Sse2, src: &[u64; 8usize]) -> [u64x2<Sse2>; 4usize] {
7557                let (chunks, []) = src.as_chunks::<2usize>() else {
7558                    unreachable!()
7559                };
7560                let v0: __m128i =
7561                    crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[0]);
7562                let v1: __m128i =
7563                    crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[1]);
7564                let v2: __m128i =
7565                    crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[2]);
7566                let v3: __m128i =
7567                    crate::transmute::checked_transmute_copy::<[u64; 2usize], __m128i>(&chunks[3]);
7568                let out0 = _mm_unpacklo_epi64(v0, v2);
7569                let out1 = _mm_unpackhi_epi64(v0, v2);
7570                let out2 = _mm_unpacklo_epi64(v1, v3);
7571                let out3 = _mm_unpackhi_epi64(v1, v3);
7572                [
7573                    out0.simd_into(token),
7574                    out1.simd_into(token),
7575                    out2.simd_into(token),
7576                    out3.simd_into(token),
7577                ]
7578            }
7579        );
7580        kernel(self, src)
7581    }
7582    #[inline(always)]
7583    fn store_four_interleaved_u64x2(
7584        self,
7585        vectors: [u64x2<Self>; 4usize],
7586        dest: &mut [u64; 8usize],
7587    ) -> () {
7588        crate::kernel!(
7589            #[inline(always)]
7590            fn kernel(token: Sse2, vectors: [u64x2<Sse2>; 4usize], dest: &mut [u64; 8usize]) -> () {
7591                let _ = token;
7592                let v0: __m128i = vectors[0].into();
7593                let v1: __m128i = vectors[1].into();
7594                let v2: __m128i = vectors[2].into();
7595                let v3: __m128i = vectors[3].into();
7596                let out0 = _mm_unpacklo_epi64(v0, v1);
7597                let out1 = _mm_unpacklo_epi64(v2, v3);
7598                let out2 = _mm_unpackhi_epi64(v0, v1);
7599                let out3 = _mm_unpackhi_epi64(v2, v3);
7600                let (chunks, []) = dest.as_chunks_mut::<2usize>() else {
7601                    unreachable!()
7602                };
7603                crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>(
7604                    out0,
7605                    &mut chunks[0],
7606                );
7607                crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>(
7608                    out1,
7609                    &mut chunks[1],
7610                );
7611                crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>(
7612                    out2,
7613                    &mut chunks[2],
7614                );
7615                crate::transmute::checked_transmute_store::<__m128i, [u64; 2usize]>(
7616                    out3,
7617                    &mut chunks[3],
7618                );
7619            }
7620        );
7621        kernel(self, vectors, dest);
7622    }
7623    #[inline(always)]
7624    fn narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
7625        crate::kernel!(
7626            #[inline(always)]
7627            fn kernel(token: Sse2, a: u64x2<Sse2>, b: u64x2<Sse2>) -> u32x4<Sse2> {
7628                _mm_castps_si128(_mm_shuffle_ps::<0x88>(
7629                    _mm_castsi128_ps(a.into()),
7630                    _mm_castsi128_ps(b.into()),
7631                ))
7632                .simd_into(token)
7633            }
7634        );
7635        kernel(self, a, b)
7636    }
7637    #[inline(always)]
7638    fn saturating_narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
7639        [
7640            a[0usize].clamp(u32::MIN as u64, u32::MAX as u64) as u32,
7641            a[1usize].clamp(u32::MIN as u64, u32::MAX as u64) as u32,
7642            b[0usize].clamp(u32::MIN as u64, u32::MAX as u64) as u32,
7643            b[1usize].clamp(u32::MIN as u64, u32::MAX as u64) as u32,
7644        ]
7645        .simd_into(self)
7646    }
7647    #[inline(always)]
7648    fn relaxed_narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
7649        debug_assert!(
7650            a.as_slice()
7651                .iter()
7652                .chain(b.as_slice())
7653                .all(|&value| { value >= u32::MIN as u64 && value <= u32::MAX as u64 }),
7654            "relaxed_narrow inputs must fit in the destination type",
7655        );
7656        self.narrow_u64x2(a, b)
7657    }
7658    #[inline(always)]
7659    fn cvt_f64_u64x2(self, a: u64x2<Self>) -> f64x2<Self> {
7660        [a[0usize] as f64, a[1usize] as f64].simd_into(self)
7661    }
7662    #[inline(always)]
7663    fn splat_mask64x2(self, val: bool) -> mask64x2<Self> {
7664        crate::kernel!(
7665            #[inline(always)]
7666            fn kernel(token: Sse2, val: bool) -> mask64x2<Sse2> {
7667                let val: i64 = if val { !0 } else { 0 };
7668                _mm_set1_epi64x(val).simd_into(token)
7669            }
7670        );
7671        kernel(self, val)
7672    }
7673    #[inline(always)]
7674    fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2<Self> {
7675        let lanes: [i64; 2usize] = [
7676            if bits & 1 != 0 { !0 } else { 0 },
7677            if (bits >> 1usize) & 1 != 0 { !0 } else { 0 },
7678        ];
7679        lanes.simd_into(self)
7680    }
7681    #[inline(always)]
7682    fn to_bitmask_mask64x2(self, a: mask64x2<Self>) -> u64 {
7683        crate::kernel!(
7684            #[inline(always)]
7685            fn kernel(token: Sse2, a: mask64x2<Sse2>) -> u64 {
7686                _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 as u64
7687            }
7688        );
7689        kernel(self, a)
7690    }
7691    #[inline(always)]
7692    fn from_vector_mask64x2(self, vector: i64x2<Self>) -> mask64x2<Self> {
7693        mask64x2 {
7694            val: vector.val,
7695            simd: self,
7696        }
7697    }
7698    #[inline(always)]
7699    fn to_vector_mask64x2(self, a: mask64x2<Self>) -> i64x2<Self> {
7700        i64x2 {
7701            val: a.val,
7702            simd: self,
7703        }
7704    }
7705    #[inline(always)]
7706    fn set_mask64x2(self, a: &mut mask64x2<Self>, index: usize, value: bool) -> () {
7707        assert!(
7708            index < 2usize,
7709            "mask lane index {index} is out of bounds for {} lanes",
7710            2usize
7711        );
7712        let mut lanes: [i64; 2usize] = (*a).into();
7713        lanes[index] = if value { !0 } else { 0 };
7714        *a = lanes.simd_into(self);
7715    }
7716    #[inline(always)]
7717    fn rotate_elements_left_mask64x2<const OFFSET: usize>(
7718        self,
7719        a: mask64x2<Self>,
7720    ) -> mask64x2<Self> {
7721        let int = i64x2 {
7722            val: crate::transmute::checked_transmute_copy(&a.val),
7723            simd: self,
7724        };
7725        let rotated = int.rotate_elements_left::<OFFSET>();
7726        mask64x2 {
7727            val: crate::transmute::checked_transmute_copy(&rotated.val),
7728            simd: self,
7729        }
7730    }
7731    #[inline(always)]
7732    fn rotate_elements_right_mask64x2<const OFFSET: usize>(
7733        self,
7734        a: mask64x2<Self>,
7735    ) -> mask64x2<Self> {
7736        let int = i64x2 {
7737            val: crate::transmute::checked_transmute_copy(&a.val),
7738            simd: self,
7739        };
7740        let rotated = int.rotate_elements_right::<OFFSET>();
7741        mask64x2 {
7742            val: crate::transmute::checked_transmute_copy(&rotated.val),
7743            simd: self,
7744        }
7745    }
7746    #[inline(always)]
7747    fn and_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7748        crate::kernel!(
7749            #[inline(always)]
7750            fn kernel(token: Sse2, a: mask64x2<Sse2>, b: mask64x2<Sse2>) -> mask64x2<Sse2> {
7751                _mm_and_si128(a.into(), b.into()).simd_into(token)
7752            }
7753        );
7754        kernel(self, a, b)
7755    }
7756    #[inline(always)]
7757    fn or_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7758        crate::kernel!(
7759            #[inline(always)]
7760            fn kernel(token: Sse2, a: mask64x2<Sse2>, b: mask64x2<Sse2>) -> mask64x2<Sse2> {
7761                _mm_or_si128(a.into(), b.into()).simd_into(token)
7762            }
7763        );
7764        kernel(self, a, b)
7765    }
7766    #[inline(always)]
7767    fn xor_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7768        crate::kernel!(
7769            #[inline(always)]
7770            fn kernel(token: Sse2, a: mask64x2<Sse2>, b: mask64x2<Sse2>) -> mask64x2<Sse2> {
7771                _mm_xor_si128(a.into(), b.into()).simd_into(token)
7772            }
7773        );
7774        kernel(self, a, b)
7775    }
7776    #[inline(always)]
7777    fn not_mask64x2(self, a: mask64x2<Self>) -> mask64x2<Self> {
7778        self.xor_mask64x2(a, self.splat_mask64x2(true))
7779    }
7780    #[inline(always)]
7781    fn reverse_mask64x2(self, a: mask64x2<Self>) -> mask64x2<Self> {
7782        let lanes = i64x2 {
7783            val: crate::transmute::checked_transmute_copy(&a.val),
7784            simd: self,
7785        };
7786        let reversed = self.reverse_i64x2(lanes);
7787        mask64x2 {
7788            val: crate::transmute::checked_transmute_copy(&reversed.val),
7789            simd: self,
7790        }
7791    }
7792    #[inline(always)]
7793    fn select_mask64x2(
7794        self,
7795        a: mask64x2<Self>,
7796        b: mask64x2<Self>,
7797        c: mask64x2<Self>,
7798    ) -> mask64x2<Self> {
7799        crate::kernel!(
7800            #[inline(always)]
7801            fn kernel(
7802                token: Sse2,
7803                a: mask64x2<Sse2>,
7804                b: mask64x2<Sse2>,
7805                c: mask64x2<Sse2>,
7806            ) -> mask64x2<Sse2> {
7807                _mm_or_si128(
7808                    _mm_and_si128(a.into(), b.into()),
7809                    _mm_andnot_si128(a.into(), c.into()),
7810                )
7811                .simd_into(token)
7812            }
7813        );
7814        kernel(self, a, b, c)
7815    }
7816    #[inline(always)]
7817    fn simd_eq_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
7818        crate::kernel!(
7819            #[inline(always)]
7820            fn kernel(token: Sse2, a: mask64x2<Sse2>, b: mask64x2<Sse2>) -> mask64x2<Sse2> {
7821                {
7822                    let eq32 = _mm_cmpeq_epi32(a.into(), b.into());
7823                    let swapped = _mm_shuffle_epi32::<0b10_11_00_01>(eq32);
7824                    _mm_and_si128(eq32, swapped)
7825                }
7826                .simd_into(token)
7827            }
7828        );
7829        kernel(self, a, b)
7830    }
7831    #[inline(always)]
7832    fn any_true_mask64x2(self, a: mask64x2<Self>) -> bool {
7833        crate::kernel!(
7834            #[inline(always)]
7835            fn kernel(token: Sse2, a: mask64x2<Sse2>) -> bool {
7836                _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 != 0
7837            }
7838        );
7839        kernel(self, a)
7840    }
7841    #[inline(always)]
7842    fn all_true_mask64x2(self, a: mask64x2<Self>) -> bool {
7843        crate::kernel!(
7844            #[inline(always)]
7845            fn kernel(token: Sse2, a: mask64x2<Sse2>) -> bool {
7846                _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 == 0b11
7847            }
7848        );
7849        kernel(self, a)
7850    }
7851    #[inline(always)]
7852    fn any_false_mask64x2(self, a: mask64x2<Self>) -> bool {
7853        crate::kernel!(
7854            #[inline(always)]
7855            fn kernel(token: Sse2, a: mask64x2<Sse2>) -> bool {
7856                _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 != 0b11
7857            }
7858        );
7859        kernel(self, a)
7860    }
7861    #[inline(always)]
7862    fn all_false_mask64x2(self, a: mask64x2<Self>) -> bool {
7863        crate::kernel!(
7864            #[inline(always)]
7865            fn kernel(token: Sse2, a: mask64x2<Sse2>) -> bool {
7866                _mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 == 0
7867            }
7868        );
7869        kernel(self, a)
7870    }
7871    #[inline(always)]
7872    fn combine_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x4<Self> {
7873        mask64x4 {
7874            val: crate::support::Aligned256([a.val.0, b.val.0]),
7875            simd: self,
7876        }
7877    }
7878    #[inline(always)]
7879    fn narrow_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask32x4<Self> {
7880        crate::kernel!(
7881            #[inline(always)]
7882            fn kernel(token: Sse2, a: mask64x2<Sse2>, b: mask64x2<Sse2>) -> mask32x4<Sse2> {
7883                _mm_castps_si128(_mm_shuffle_ps::<0x88>(
7884                    _mm_castsi128_ps(a.into()),
7885                    _mm_castsi128_ps(b.into()),
7886                ))
7887                .simd_into(token)
7888            }
7889        );
7890        kernel(self, a, b)
7891    }
7892    #[inline(always)]
7893    fn slide_f32x8<const SHIFT: usize>(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
7894        if SHIFT >= 8usize {
7895            return b;
7896        }
7897        let result = cross_block_alignr_128x2(
7898            self,
7899            Bytes::to_bytes(b).val.0,
7900            Bytes::to_bytes(a).val.0,
7901            SHIFT * 4usize,
7902        );
7903        Bytes::from_bytes(u8x32 {
7904            val: crate::support::Aligned256(result),
7905            simd: self,
7906        })
7907    }
7908    #[inline(always)]
7909    fn compress_f32x8(self, values: f32x8<Self>, mask: mask32x8<Self>) -> f32x8<Self> {
7910        let mask = mask8x32 {
7911            val: crate::transmute::checked_transmute_copy(&mask.val),
7912            simd: self,
7913        };
7914        Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
7915    }
7916    #[inline(always)]
7917    fn compress_merge_f32x8(
7918        self,
7919        values: f32x8<Self>,
7920        mask: mask32x8<Self>,
7921        merge: f32x8<Self>,
7922    ) -> f32x8<Self> {
7923        let mask = mask8x32 {
7924            val: crate::transmute::checked_transmute_copy(&mask.val),
7925            simd: self,
7926        };
7927        Bytes::from_bytes(self.compress_merge_u8x32(
7928            Bytes::to_bytes(values),
7929            mask,
7930            Bytes::to_bytes(merge),
7931        ))
7932    }
7933    #[inline(always)]
7934    fn expand_f32x8(self, values: f32x8<Self>, mask: mask32x8<Self>) -> f32x8<Self> {
7935        let mask = mask8x32 {
7936            val: crate::transmute::checked_transmute_copy(&mask.val),
7937            simd: self,
7938        };
7939        Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
7940    }
7941    #[inline(always)]
7942    fn expand_merge_f32x8(
7943        self,
7944        values: f32x8<Self>,
7945        mask: mask32x8<Self>,
7946        merge: f32x8<Self>,
7947    ) -> f32x8<Self> {
7948        let mask = mask8x32 {
7949            val: crate::transmute::checked_transmute_copy(&mask.val),
7950            simd: self,
7951        };
7952        Bytes::from_bytes(self.expand_merge_u8x32(
7953            Bytes::to_bytes(values),
7954            mask,
7955            Bytes::to_bytes(merge),
7956        ))
7957    }
7958    #[inline(always)]
7959    fn simd_ne_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
7960        let (a0, a1) = self.split_f32x8(a);
7961        let (b0, b1) = self.split_f32x8(b);
7962        self.combine_mask32x4(self.simd_ne_f32x4(a0, b0), self.simd_ne_f32x4(a1, b1))
7963    }
7964    #[inline(always)]
7965    fn is_nan_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7966        a.simd_ne(a)
7967    }
7968    #[inline(always)]
7969    fn is_infinite_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7970        a.abs().simd_eq(f32::INFINITY)
7971    }
7972    #[inline(always)]
7973    fn is_finite_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7974        a.abs().simd_lt(f32::INFINITY)
7975    }
7976    #[inline(always)]
7977    fn is_subnormal_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7978        let i: i32x8<Self> = a.bitcast();
7979        (i & 0x7F80_0000_i32).simd_eq(0) & (i & 0x007F_FFFF_i32).simd_ne(0)
7980    }
7981    #[inline(always)]
7982    fn is_normal_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7983        let i: i32x8<Self> = a.bitcast();
7984        let exp = i & 0x7F80_0000_i32;
7985        exp.simd_ne(0) & exp.simd_ne(0x7F80_0000_i32)
7986    }
7987    #[inline(always)]
7988    fn is_sign_positive_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7989        let i: i32x8<Self> = a.bitcast();
7990        (i & i32::MIN).simd_eq(0)
7991    }
7992    #[inline(always)]
7993    fn is_sign_negative_f32x8(self, a: f32x8<Self>) -> mask32x8<Self> {
7994        !a.is_sign_positive()
7995    }
7996    #[inline(always)]
7997    fn combine_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x16<Self> {
7998        f32x16 {
7999            val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8000            simd: self,
8001        }
8002    }
8003    #[inline(always)]
8004    fn split_f32x8(self, a: f32x8<Self>) -> (f32x4<Self>, f32x4<Self>) {
8005        (
8006            f32x4 {
8007                val: crate::support::Aligned128(a.val.0[0]),
8008                simd: self,
8009            },
8010            f32x4 {
8011                val: crate::support::Aligned128(a.val.0[1]),
8012                simd: self,
8013            },
8014        )
8015    }
8016    #[inline(always)]
8017    fn slide_i8x32<const SHIFT: usize>(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
8018        if SHIFT >= 32usize {
8019            return b;
8020        }
8021        let result = cross_block_alignr_128x2(
8022            self,
8023            Bytes::to_bytes(b).val.0,
8024            Bytes::to_bytes(a).val.0,
8025            SHIFT,
8026        );
8027        Bytes::from_bytes(u8x32 {
8028            val: crate::support::Aligned256(result),
8029            simd: self,
8030        })
8031    }
8032    #[inline(always)]
8033    fn compress_i8x32(self, values: i8x32<Self>, mask: mask8x32<Self>) -> i8x32<Self> {
8034        let mask = mask8x32 {
8035            val: crate::transmute::checked_transmute_copy(&mask.val),
8036            simd: self,
8037        };
8038        Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
8039    }
8040    #[inline(always)]
8041    fn compress_merge_i8x32(
8042        self,
8043        values: i8x32<Self>,
8044        mask: mask8x32<Self>,
8045        merge: i8x32<Self>,
8046    ) -> i8x32<Self> {
8047        let mask = mask8x32 {
8048            val: crate::transmute::checked_transmute_copy(&mask.val),
8049            simd: self,
8050        };
8051        Bytes::from_bytes(self.compress_merge_u8x32(
8052            Bytes::to_bytes(values),
8053            mask,
8054            Bytes::to_bytes(merge),
8055        ))
8056    }
8057    #[inline(always)]
8058    fn expand_i8x32(self, values: i8x32<Self>, mask: mask8x32<Self>) -> i8x32<Self> {
8059        let mask = mask8x32 {
8060            val: crate::transmute::checked_transmute_copy(&mask.val),
8061            simd: self,
8062        };
8063        Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
8064    }
8065    #[inline(always)]
8066    fn expand_merge_i8x32(
8067        self,
8068        values: i8x32<Self>,
8069        mask: mask8x32<Self>,
8070        merge: i8x32<Self>,
8071    ) -> i8x32<Self> {
8072        let mask = mask8x32 {
8073            val: crate::transmute::checked_transmute_copy(&mask.val),
8074            simd: self,
8075        };
8076        Bytes::from_bytes(self.expand_merge_u8x32(
8077            Bytes::to_bytes(values),
8078            mask,
8079            Bytes::to_bytes(merge),
8080        ))
8081    }
8082    #[inline(always)]
8083    fn combine_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x64<Self> {
8084        i8x64 {
8085            val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8086            simd: self,
8087        }
8088    }
8089    #[inline(always)]
8090    fn split_i8x32(self, a: i8x32<Self>) -> (i8x16<Self>, i8x16<Self>) {
8091        (
8092            i8x16 {
8093                val: crate::support::Aligned128(a.val.0[0]),
8094                simd: self,
8095            },
8096            i8x16 {
8097                val: crate::support::Aligned128(a.val.0[1]),
8098                simd: self,
8099            },
8100        )
8101    }
8102    #[inline(always)]
8103    fn slide_u8x32<const SHIFT: usize>(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
8104        if SHIFT >= 32usize {
8105            return b;
8106        }
8107        let result = cross_block_alignr_128x2(
8108            self,
8109            Bytes::to_bytes(b).val.0,
8110            Bytes::to_bytes(a).val.0,
8111            SHIFT,
8112        );
8113        Bytes::from_bytes(u8x32 {
8114            val: crate::support::Aligned256(result),
8115            simd: self,
8116        })
8117    }
8118    #[inline(always)]
8119    fn swizzle_dyn_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
8120        let bytes = Bytes::to_bytes(a);
8121        let mut output = [0u8; 32usize];
8122        for lane in 0..32usize {
8123            let index = indices[lane] as usize;
8124            output[lane] = bytes[index % 32usize];
8125        }
8126        let result: u8x32<Self> = output.simd_into(self);
8127        Bytes::from_bytes(result)
8128    }
8129    #[inline(always)]
8130    fn swizzle_dyn_precise_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
8131        let bytes = Bytes::to_bytes(a);
8132        let mut output = [0u8; 32usize];
8133        for lane in 0..32usize {
8134            let index = indices[lane] as usize;
8135            let value = bytes[index % 32usize];
8136            output[lane] = core::hint::select_unpredictable(index < 32usize, value, 0u8);
8137        }
8138        let result: u8x32<Self> = output.simd_into(self);
8139        Bytes::from_bytes(result)
8140    }
8141    #[inline(always)]
8142    fn concat_swizzle_dyn_u8x32(
8143        self,
8144        a: u8x32<Self>,
8145        b: u8x32<Self>,
8146        indices: u8x32<Self>,
8147    ) -> u8x32<Self> {
8148        let first_table = Bytes::to_bytes(a);
8149        let second_table = Bytes::to_bytes(b);
8150        let mut output = [0u8; 32usize];
8151        for lane in 0..32usize {
8152            let index = indices[lane] as usize % 64usize;
8153            output[lane] = if index < 32usize {
8154                first_table[index]
8155            } else {
8156                second_table[index - 32usize]
8157            };
8158        }
8159        let result: u8x32<Self> = output.simd_into(self);
8160        Bytes::from_bytes(result)
8161    }
8162    #[inline(always)]
8163    fn concat_swizzle_dyn_precise_u8x32(
8164        self,
8165        a: u8x32<Self>,
8166        b: u8x32<Self>,
8167        indices: u8x32<Self>,
8168    ) -> u8x32<Self> {
8169        let first_table = Bytes::to_bytes(a);
8170        let second_table = Bytes::to_bytes(b);
8171        let mut output = [0u8; 32usize];
8172        for lane in 0..32usize {
8173            let index = indices[lane] as usize;
8174            let table_index = index % 32usize;
8175            let value = if index < 32usize {
8176                first_table[table_index]
8177            } else {
8178                second_table[table_index]
8179            };
8180            output[lane] = if index < 64usize { value } else { 0 };
8181        }
8182        let result: u8x32<Self> = output.simd_into(self);
8183        Bytes::from_bytes(result)
8184    }
8185    #[inline(always)]
8186    fn compress_u8x32(self, values: u8x32<Self>, mask: mask8x32<Self>) -> u8x32<Self> {
8187        self.compress_merge_u8x32(values, mask, u8x32::splat(self, 0))
8188    }
8189    #[inline(always)]
8190    fn compress_merge_u8x32(
8191        self,
8192        values: u8x32<Self>,
8193        mask: mask8x32<Self>,
8194        merge: u8x32<Self>,
8195    ) -> u8x32<Self> {
8196        let mask = self.to_bitmask_mask8x32(mask);
8197        let mut merge_padded = [0u8; 32 + 1];
8198        merge_padded[..32].copy_from_slice(&*merge);
8199        let mut compacted = merge_padded;
8200        let mut output_lane = 0;
8201        for input_lane in 0..32 {
8202            compacted[output_lane] = values[input_lane];
8203            output_lane += ((mask >> input_lane) & 1) as usize;
8204        }
8205        compacted[output_lane] = merge_padded[output_lane];
8206        let mut result = merge;
8207        result.copy_from_slice(&compacted[..32]);
8208        result
8209    }
8210    #[inline(always)]
8211    fn expand_u8x32(self, values: u8x32<Self>, mask: mask8x32<Self>) -> u8x32<Self> {
8212        self.expand_merge_u8x32(values, mask, u8x32::splat(self, 0))
8213    }
8214    #[inline(always)]
8215    fn expand_merge_u8x32(
8216        self,
8217        values: u8x32<Self>,
8218        mask: mask8x32<Self>,
8219        merge: u8x32<Self>,
8220    ) -> u8x32<Self> {
8221        let mask = self.to_bitmask_mask8x32(mask);
8222        let mut result = merge;
8223        let mut input_lane = 0;
8224        for output_lane in 0..32 {
8225            let bit = ((mask >> output_lane) & 1) as u8;
8226            let keep = 0u8.wrapping_sub(bit);
8227            result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep);
8228            input_lane += usize::from(bit);
8229        }
8230        result
8231    }
8232    #[inline(always)]
8233    fn combine_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x64<Self> {
8234        u8x64 {
8235            val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8236            simd: self,
8237        }
8238    }
8239    #[inline(always)]
8240    fn split_u8x32(self, a: u8x32<Self>) -> (u8x16<Self>, u8x16<Self>) {
8241        (
8242            u8x16 {
8243                val: crate::support::Aligned128(a.val.0[0]),
8244                simd: self,
8245            },
8246            u8x16 {
8247                val: crate::support::Aligned128(a.val.0[1]),
8248                simd: self,
8249            },
8250        )
8251    }
8252    #[inline(always)]
8253    fn from_vector_mask8x32(self, vector: i8x32<Self>) -> mask8x32<Self> {
8254        mask8x32 {
8255            val: vector.val,
8256            simd: self,
8257        }
8258    }
8259    #[inline(always)]
8260    fn to_vector_mask8x32(self, a: mask8x32<Self>) -> i8x32<Self> {
8261        i8x32 {
8262            val: a.val,
8263            simd: self,
8264        }
8265    }
8266    #[inline(always)]
8267    fn set_mask8x32(self, a: &mut mask8x32<Self>, index: usize, value: bool) -> () {
8268        assert!(
8269            index < 32usize,
8270            "mask lane index {index} is out of bounds for {} lanes",
8271            32usize
8272        );
8273        let mut lanes: [i8; 32usize] = (*a).into();
8274        lanes[index] = if value { !0 } else { 0 };
8275        *a = lanes.simd_into(self);
8276    }
8277    #[inline(always)]
8278    fn rotate_elements_left_mask8x32<const OFFSET: usize>(
8279        self,
8280        a: mask8x32<Self>,
8281    ) -> mask8x32<Self> {
8282        let int = i8x32 {
8283            val: crate::transmute::checked_transmute_copy(&a.val),
8284            simd: self,
8285        };
8286        let rotated = int.rotate_elements_left::<OFFSET>();
8287        mask8x32 {
8288            val: crate::transmute::checked_transmute_copy(&rotated.val),
8289            simd: self,
8290        }
8291    }
8292    #[inline(always)]
8293    fn rotate_elements_right_mask8x32<const OFFSET: usize>(
8294        self,
8295        a: mask8x32<Self>,
8296    ) -> mask8x32<Self> {
8297        let int = i8x32 {
8298            val: crate::transmute::checked_transmute_copy(&a.val),
8299            simd: self,
8300        };
8301        let rotated = int.rotate_elements_right::<OFFSET>();
8302        mask8x32 {
8303            val: crate::transmute::checked_transmute_copy(&rotated.val),
8304            simd: self,
8305        }
8306    }
8307    #[inline(always)]
8308    fn combine_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x64<Self> {
8309        mask8x64 {
8310            val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8311            simd: self,
8312        }
8313    }
8314    #[inline(always)]
8315    fn split_mask8x32(self, a: mask8x32<Self>) -> (mask8x16<Self>, mask8x16<Self>) {
8316        (
8317            mask8x16 {
8318                val: crate::support::Aligned128(a.val.0[0]),
8319                simd: self,
8320            },
8321            mask8x16 {
8322                val: crate::support::Aligned128(a.val.0[1]),
8323                simd: self,
8324            },
8325        )
8326    }
8327    #[inline(always)]
8328    fn slide_i16x16<const SHIFT: usize>(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
8329        if SHIFT >= 16usize {
8330            return b;
8331        }
8332        let result = cross_block_alignr_128x2(
8333            self,
8334            Bytes::to_bytes(b).val.0,
8335            Bytes::to_bytes(a).val.0,
8336            SHIFT * 2usize,
8337        );
8338        Bytes::from_bytes(u8x32 {
8339            val: crate::support::Aligned256(result),
8340            simd: self,
8341        })
8342    }
8343    #[inline(always)]
8344    fn compress_i16x16(self, values: i16x16<Self>, mask: mask16x16<Self>) -> i16x16<Self> {
8345        let mask = mask8x32 {
8346            val: crate::transmute::checked_transmute_copy(&mask.val),
8347            simd: self,
8348        };
8349        Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
8350    }
8351    #[inline(always)]
8352    fn compress_merge_i16x16(
8353        self,
8354        values: i16x16<Self>,
8355        mask: mask16x16<Self>,
8356        merge: i16x16<Self>,
8357    ) -> i16x16<Self> {
8358        let mask = mask8x32 {
8359            val: crate::transmute::checked_transmute_copy(&mask.val),
8360            simd: self,
8361        };
8362        Bytes::from_bytes(self.compress_merge_u8x32(
8363            Bytes::to_bytes(values),
8364            mask,
8365            Bytes::to_bytes(merge),
8366        ))
8367    }
8368    #[inline(always)]
8369    fn expand_i16x16(self, values: i16x16<Self>, mask: mask16x16<Self>) -> i16x16<Self> {
8370        let mask = mask8x32 {
8371            val: crate::transmute::checked_transmute_copy(&mask.val),
8372            simd: self,
8373        };
8374        Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
8375    }
8376    #[inline(always)]
8377    fn expand_merge_i16x16(
8378        self,
8379        values: i16x16<Self>,
8380        mask: mask16x16<Self>,
8381        merge: i16x16<Self>,
8382    ) -> i16x16<Self> {
8383        let mask = mask8x32 {
8384            val: crate::transmute::checked_transmute_copy(&mask.val),
8385            simd: self,
8386        };
8387        Bytes::from_bytes(self.expand_merge_u8x32(
8388            Bytes::to_bytes(values),
8389            mask,
8390            Bytes::to_bytes(merge),
8391        ))
8392    }
8393    #[inline(always)]
8394    fn combine_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x32<Self> {
8395        i16x32 {
8396            val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8397            simd: self,
8398        }
8399    }
8400    #[inline(always)]
8401    fn split_i16x16(self, a: i16x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
8402        (
8403            i16x8 {
8404                val: crate::support::Aligned128(a.val.0[0]),
8405                simd: self,
8406            },
8407            i16x8 {
8408                val: crate::support::Aligned128(a.val.0[1]),
8409                simd: self,
8410            },
8411        )
8412    }
8413    #[inline(always)]
8414    fn slide_u16x16<const SHIFT: usize>(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
8415        if SHIFT >= 16usize {
8416            return b;
8417        }
8418        let result = cross_block_alignr_128x2(
8419            self,
8420            Bytes::to_bytes(b).val.0,
8421            Bytes::to_bytes(a).val.0,
8422            SHIFT * 2usize,
8423        );
8424        Bytes::from_bytes(u8x32 {
8425            val: crate::support::Aligned256(result),
8426            simd: self,
8427        })
8428    }
8429    #[inline(always)]
8430    fn compress_u16x16(self, values: u16x16<Self>, mask: mask16x16<Self>) -> u16x16<Self> {
8431        let mask = mask8x32 {
8432            val: crate::transmute::checked_transmute_copy(&mask.val),
8433            simd: self,
8434        };
8435        Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
8436    }
8437    #[inline(always)]
8438    fn compress_merge_u16x16(
8439        self,
8440        values: u16x16<Self>,
8441        mask: mask16x16<Self>,
8442        merge: u16x16<Self>,
8443    ) -> u16x16<Self> {
8444        let mask = mask8x32 {
8445            val: crate::transmute::checked_transmute_copy(&mask.val),
8446            simd: self,
8447        };
8448        Bytes::from_bytes(self.compress_merge_u8x32(
8449            Bytes::to_bytes(values),
8450            mask,
8451            Bytes::to_bytes(merge),
8452        ))
8453    }
8454    #[inline(always)]
8455    fn expand_u16x16(self, values: u16x16<Self>, mask: mask16x16<Self>) -> u16x16<Self> {
8456        let mask = mask8x32 {
8457            val: crate::transmute::checked_transmute_copy(&mask.val),
8458            simd: self,
8459        };
8460        Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
8461    }
8462    #[inline(always)]
8463    fn expand_merge_u16x16(
8464        self,
8465        values: u16x16<Self>,
8466        mask: mask16x16<Self>,
8467        merge: u16x16<Self>,
8468    ) -> u16x16<Self> {
8469        let mask = mask8x32 {
8470            val: crate::transmute::checked_transmute_copy(&mask.val),
8471            simd: self,
8472        };
8473        Bytes::from_bytes(self.expand_merge_u8x32(
8474            Bytes::to_bytes(values),
8475            mask,
8476            Bytes::to_bytes(merge),
8477        ))
8478    }
8479    #[inline(always)]
8480    fn combine_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x32<Self> {
8481        u16x32 {
8482            val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8483            simd: self,
8484        }
8485    }
8486    #[inline(always)]
8487    fn split_u16x16(self, a: u16x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
8488        (
8489            u16x8 {
8490                val: crate::support::Aligned128(a.val.0[0]),
8491                simd: self,
8492            },
8493            u16x8 {
8494                val: crate::support::Aligned128(a.val.0[1]),
8495                simd: self,
8496            },
8497        )
8498    }
8499    #[inline(always)]
8500    fn to_bitmask_mask16x16(self, a: mask16x16<Self>) -> u64 {
8501        crate::kernel!(
8502            #[inline(always)]
8503            fn kernel(token: Sse2, a: mask16x16<Sse2>) -> u64 {
8504                {
8505                    let packed = _mm_packs_epi16(a.val.0[0], a.val.0[1]);
8506                    _mm_movemask_epi8(packed) as u32 as u64
8507                }
8508            }
8509        );
8510        kernel(self, a)
8511    }
8512    #[inline(always)]
8513    fn from_vector_mask16x16(self, vector: i16x16<Self>) -> mask16x16<Self> {
8514        mask16x16 {
8515            val: vector.val,
8516            simd: self,
8517        }
8518    }
8519    #[inline(always)]
8520    fn to_vector_mask16x16(self, a: mask16x16<Self>) -> i16x16<Self> {
8521        i16x16 {
8522            val: a.val,
8523            simd: self,
8524        }
8525    }
8526    #[inline(always)]
8527    fn set_mask16x16(self, a: &mut mask16x16<Self>, index: usize, value: bool) -> () {
8528        assert!(
8529            index < 16usize,
8530            "mask lane index {index} is out of bounds for {} lanes",
8531            16usize
8532        );
8533        let mut lanes: [i16; 16usize] = (*a).into();
8534        lanes[index] = if value { !0 } else { 0 };
8535        *a = lanes.simd_into(self);
8536    }
8537    #[inline(always)]
8538    fn rotate_elements_left_mask16x16<const OFFSET: usize>(
8539        self,
8540        a: mask16x16<Self>,
8541    ) -> mask16x16<Self> {
8542        let int = i16x16 {
8543            val: crate::transmute::checked_transmute_copy(&a.val),
8544            simd: self,
8545        };
8546        let rotated = int.rotate_elements_left::<OFFSET>();
8547        mask16x16 {
8548            val: crate::transmute::checked_transmute_copy(&rotated.val),
8549            simd: self,
8550        }
8551    }
8552    #[inline(always)]
8553    fn rotate_elements_right_mask16x16<const OFFSET: usize>(
8554        self,
8555        a: mask16x16<Self>,
8556    ) -> mask16x16<Self> {
8557        let int = i16x16 {
8558            val: crate::transmute::checked_transmute_copy(&a.val),
8559            simd: self,
8560        };
8561        let rotated = int.rotate_elements_right::<OFFSET>();
8562        mask16x16 {
8563            val: crate::transmute::checked_transmute_copy(&rotated.val),
8564            simd: self,
8565        }
8566    }
8567    #[inline(always)]
8568    fn combine_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x32<Self> {
8569        mask16x32 {
8570            val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8571            simd: self,
8572        }
8573    }
8574    #[inline(always)]
8575    fn split_mask16x16(self, a: mask16x16<Self>) -> (mask16x8<Self>, mask16x8<Self>) {
8576        (
8577            mask16x8 {
8578                val: crate::support::Aligned128(a.val.0[0]),
8579                simd: self,
8580            },
8581            mask16x8 {
8582                val: crate::support::Aligned128(a.val.0[1]),
8583                simd: self,
8584            },
8585        )
8586    }
8587    #[inline(always)]
8588    fn slide_i32x8<const SHIFT: usize>(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
8589        if SHIFT >= 8usize {
8590            return b;
8591        }
8592        let result = cross_block_alignr_128x2(
8593            self,
8594            Bytes::to_bytes(b).val.0,
8595            Bytes::to_bytes(a).val.0,
8596            SHIFT * 4usize,
8597        );
8598        Bytes::from_bytes(u8x32 {
8599            val: crate::support::Aligned256(result),
8600            simd: self,
8601        })
8602    }
8603    #[inline(always)]
8604    fn compress_i32x8(self, values: i32x8<Self>, mask: mask32x8<Self>) -> i32x8<Self> {
8605        let mask = mask8x32 {
8606            val: crate::transmute::checked_transmute_copy(&mask.val),
8607            simd: self,
8608        };
8609        Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
8610    }
8611    #[inline(always)]
8612    fn compress_merge_i32x8(
8613        self,
8614        values: i32x8<Self>,
8615        mask: mask32x8<Self>,
8616        merge: i32x8<Self>,
8617    ) -> i32x8<Self> {
8618        let mask = mask8x32 {
8619            val: crate::transmute::checked_transmute_copy(&mask.val),
8620            simd: self,
8621        };
8622        Bytes::from_bytes(self.compress_merge_u8x32(
8623            Bytes::to_bytes(values),
8624            mask,
8625            Bytes::to_bytes(merge),
8626        ))
8627    }
8628    #[inline(always)]
8629    fn expand_i32x8(self, values: i32x8<Self>, mask: mask32x8<Self>) -> i32x8<Self> {
8630        let mask = mask8x32 {
8631            val: crate::transmute::checked_transmute_copy(&mask.val),
8632            simd: self,
8633        };
8634        Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
8635    }
8636    #[inline(always)]
8637    fn expand_merge_i32x8(
8638        self,
8639        values: i32x8<Self>,
8640        mask: mask32x8<Self>,
8641        merge: i32x8<Self>,
8642    ) -> i32x8<Self> {
8643        let mask = mask8x32 {
8644            val: crate::transmute::checked_transmute_copy(&mask.val),
8645            simd: self,
8646        };
8647        Bytes::from_bytes(self.expand_merge_u8x32(
8648            Bytes::to_bytes(values),
8649            mask,
8650            Bytes::to_bytes(merge),
8651        ))
8652    }
8653    #[inline(always)]
8654    fn combine_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x16<Self> {
8655        i32x16 {
8656            val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8657            simd: self,
8658        }
8659    }
8660    #[inline(always)]
8661    fn split_i32x8(self, a: i32x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
8662        (
8663            i32x4 {
8664                val: crate::support::Aligned128(a.val.0[0]),
8665                simd: self,
8666            },
8667            i32x4 {
8668                val: crate::support::Aligned128(a.val.0[1]),
8669                simd: self,
8670            },
8671        )
8672    }
8673    #[inline(always)]
8674    fn slide_u32x8<const SHIFT: usize>(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
8675        if SHIFT >= 8usize {
8676            return b;
8677        }
8678        let result = cross_block_alignr_128x2(
8679            self,
8680            Bytes::to_bytes(b).val.0,
8681            Bytes::to_bytes(a).val.0,
8682            SHIFT * 4usize,
8683        );
8684        Bytes::from_bytes(u8x32 {
8685            val: crate::support::Aligned256(result),
8686            simd: self,
8687        })
8688    }
8689    #[inline(always)]
8690    fn compress_u32x8(self, values: u32x8<Self>, mask: mask32x8<Self>) -> u32x8<Self> {
8691        let mask = mask8x32 {
8692            val: crate::transmute::checked_transmute_copy(&mask.val),
8693            simd: self,
8694        };
8695        Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
8696    }
8697    #[inline(always)]
8698    fn compress_merge_u32x8(
8699        self,
8700        values: u32x8<Self>,
8701        mask: mask32x8<Self>,
8702        merge: u32x8<Self>,
8703    ) -> u32x8<Self> {
8704        let mask = mask8x32 {
8705            val: crate::transmute::checked_transmute_copy(&mask.val),
8706            simd: self,
8707        };
8708        Bytes::from_bytes(self.compress_merge_u8x32(
8709            Bytes::to_bytes(values),
8710            mask,
8711            Bytes::to_bytes(merge),
8712        ))
8713    }
8714    #[inline(always)]
8715    fn expand_u32x8(self, values: u32x8<Self>, mask: mask32x8<Self>) -> u32x8<Self> {
8716        let mask = mask8x32 {
8717            val: crate::transmute::checked_transmute_copy(&mask.val),
8718            simd: self,
8719        };
8720        Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
8721    }
8722    #[inline(always)]
8723    fn expand_merge_u32x8(
8724        self,
8725        values: u32x8<Self>,
8726        mask: mask32x8<Self>,
8727        merge: u32x8<Self>,
8728    ) -> u32x8<Self> {
8729        let mask = mask8x32 {
8730            val: crate::transmute::checked_transmute_copy(&mask.val),
8731            simd: self,
8732        };
8733        Bytes::from_bytes(self.expand_merge_u8x32(
8734            Bytes::to_bytes(values),
8735            mask,
8736            Bytes::to_bytes(merge),
8737        ))
8738    }
8739    #[inline(always)]
8740    fn combine_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x16<Self> {
8741        u32x16 {
8742            val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8743            simd: self,
8744        }
8745    }
8746    #[inline(always)]
8747    fn split_u32x8(self, a: u32x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
8748        (
8749            u32x4 {
8750                val: crate::support::Aligned128(a.val.0[0]),
8751                simd: self,
8752            },
8753            u32x4 {
8754                val: crate::support::Aligned128(a.val.0[1]),
8755                simd: self,
8756            },
8757        )
8758    }
8759    #[inline(always)]
8760    fn from_vector_mask32x8(self, vector: i32x8<Self>) -> mask32x8<Self> {
8761        mask32x8 {
8762            val: vector.val,
8763            simd: self,
8764        }
8765    }
8766    #[inline(always)]
8767    fn to_vector_mask32x8(self, a: mask32x8<Self>) -> i32x8<Self> {
8768        i32x8 {
8769            val: a.val,
8770            simd: self,
8771        }
8772    }
8773    #[inline(always)]
8774    fn set_mask32x8(self, a: &mut mask32x8<Self>, index: usize, value: bool) -> () {
8775        assert!(
8776            index < 8usize,
8777            "mask lane index {index} is out of bounds for {} lanes",
8778            8usize
8779        );
8780        let mut lanes: [i32; 8usize] = (*a).into();
8781        lanes[index] = if value { !0 } else { 0 };
8782        *a = lanes.simd_into(self);
8783    }
8784    #[inline(always)]
8785    fn rotate_elements_left_mask32x8<const OFFSET: usize>(
8786        self,
8787        a: mask32x8<Self>,
8788    ) -> mask32x8<Self> {
8789        let int = i32x8 {
8790            val: crate::transmute::checked_transmute_copy(&a.val),
8791            simd: self,
8792        };
8793        let rotated = int.rotate_elements_left::<OFFSET>();
8794        mask32x8 {
8795            val: crate::transmute::checked_transmute_copy(&rotated.val),
8796            simd: self,
8797        }
8798    }
8799    #[inline(always)]
8800    fn rotate_elements_right_mask32x8<const OFFSET: usize>(
8801        self,
8802        a: mask32x8<Self>,
8803    ) -> mask32x8<Self> {
8804        let int = i32x8 {
8805            val: crate::transmute::checked_transmute_copy(&a.val),
8806            simd: self,
8807        };
8808        let rotated = int.rotate_elements_right::<OFFSET>();
8809        mask32x8 {
8810            val: crate::transmute::checked_transmute_copy(&rotated.val),
8811            simd: self,
8812        }
8813    }
8814    #[inline(always)]
8815    fn combine_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x16<Self> {
8816        mask32x16 {
8817            val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8818            simd: self,
8819        }
8820    }
8821    #[inline(always)]
8822    fn split_mask32x8(self, a: mask32x8<Self>) -> (mask32x4<Self>, mask32x4<Self>) {
8823        (
8824            mask32x4 {
8825                val: crate::support::Aligned128(a.val.0[0]),
8826                simd: self,
8827            },
8828            mask32x4 {
8829                val: crate::support::Aligned128(a.val.0[1]),
8830                simd: self,
8831            },
8832        )
8833    }
8834    #[inline(always)]
8835    fn slide_f64x4<const SHIFT: usize>(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
8836        if SHIFT >= 4usize {
8837            return b;
8838        }
8839        let result = cross_block_alignr_128x2(
8840            self,
8841            Bytes::to_bytes(b).val.0,
8842            Bytes::to_bytes(a).val.0,
8843            SHIFT * 8usize,
8844        );
8845        Bytes::from_bytes(u8x32 {
8846            val: crate::support::Aligned256(result),
8847            simd: self,
8848        })
8849    }
8850    #[inline(always)]
8851    fn compress_f64x4(self, values: f64x4<Self>, mask: mask64x4<Self>) -> f64x4<Self> {
8852        let mask = mask8x32 {
8853            val: crate::transmute::checked_transmute_copy(&mask.val),
8854            simd: self,
8855        };
8856        Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
8857    }
8858    #[inline(always)]
8859    fn compress_merge_f64x4(
8860        self,
8861        values: f64x4<Self>,
8862        mask: mask64x4<Self>,
8863        merge: f64x4<Self>,
8864    ) -> f64x4<Self> {
8865        let mask = mask8x32 {
8866            val: crate::transmute::checked_transmute_copy(&mask.val),
8867            simd: self,
8868        };
8869        Bytes::from_bytes(self.compress_merge_u8x32(
8870            Bytes::to_bytes(values),
8871            mask,
8872            Bytes::to_bytes(merge),
8873        ))
8874    }
8875    #[inline(always)]
8876    fn expand_f64x4(self, values: f64x4<Self>, mask: mask64x4<Self>) -> f64x4<Self> {
8877        let mask = mask8x32 {
8878            val: crate::transmute::checked_transmute_copy(&mask.val),
8879            simd: self,
8880        };
8881        Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
8882    }
8883    #[inline(always)]
8884    fn expand_merge_f64x4(
8885        self,
8886        values: f64x4<Self>,
8887        mask: mask64x4<Self>,
8888        merge: f64x4<Self>,
8889    ) -> f64x4<Self> {
8890        let mask = mask8x32 {
8891            val: crate::transmute::checked_transmute_copy(&mask.val),
8892            simd: self,
8893        };
8894        Bytes::from_bytes(self.expand_merge_u8x32(
8895            Bytes::to_bytes(values),
8896            mask,
8897            Bytes::to_bytes(merge),
8898        ))
8899    }
8900    #[inline(always)]
8901    fn simd_ne_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
8902        let (a0, a1) = self.split_f64x4(a);
8903        let (b0, b1) = self.split_f64x4(b);
8904        self.combine_mask64x2(self.simd_ne_f64x2(a0, b0), self.simd_ne_f64x2(a1, b1))
8905    }
8906    #[inline(always)]
8907    fn is_nan_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8908        a.simd_ne(a)
8909    }
8910    #[inline(always)]
8911    fn is_infinite_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8912        a.abs().simd_eq(f64::INFINITY)
8913    }
8914    #[inline(always)]
8915    fn is_finite_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8916        a.abs().simd_lt(f64::INFINITY)
8917    }
8918    #[inline(always)]
8919    fn is_subnormal_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8920        let i: i64x4<Self> = a.bitcast();
8921        (i & 0x7FF0_0000_0000_0000_i64).simd_eq(0) & (i & 0x000F_FFFF_FFFF_FFFF_i64).simd_ne(0)
8922    }
8923    #[inline(always)]
8924    fn is_normal_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8925        let i: i64x4<Self> = a.bitcast();
8926        let exp = i & 0x7FF0_0000_0000_0000_i64;
8927        exp.simd_ne(0) & exp.simd_ne(0x7FF0_0000_0000_0000_i64)
8928    }
8929    #[inline(always)]
8930    fn is_sign_positive_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8931        let i: i64x4<Self> = a.bitcast();
8932        (i & i64::MIN).simd_eq(0)
8933    }
8934    #[inline(always)]
8935    fn is_sign_negative_f64x4(self, a: f64x4<Self>) -> mask64x4<Self> {
8936        !a.is_sign_positive()
8937    }
8938    #[inline(always)]
8939    fn combine_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x8<Self> {
8940        f64x8 {
8941            val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
8942            simd: self,
8943        }
8944    }
8945    #[inline(always)]
8946    fn split_f64x4(self, a: f64x4<Self>) -> (f64x2<Self>, f64x2<Self>) {
8947        (
8948            f64x2 {
8949                val: crate::support::Aligned128(a.val.0[0]),
8950                simd: self,
8951            },
8952            f64x2 {
8953                val: crate::support::Aligned128(a.val.0[1]),
8954                simd: self,
8955            },
8956        )
8957    }
8958    #[inline(always)]
8959    fn slide_i64x4<const SHIFT: usize>(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
8960        if SHIFT >= 4usize {
8961            return b;
8962        }
8963        let result = cross_block_alignr_128x2(
8964            self,
8965            Bytes::to_bytes(b).val.0,
8966            Bytes::to_bytes(a).val.0,
8967            SHIFT * 8usize,
8968        );
8969        Bytes::from_bytes(u8x32 {
8970            val: crate::support::Aligned256(result),
8971            simd: self,
8972        })
8973    }
8974    #[inline(always)]
8975    fn compress_i64x4(self, values: i64x4<Self>, mask: mask64x4<Self>) -> i64x4<Self> {
8976        let mask = mask8x32 {
8977            val: crate::transmute::checked_transmute_copy(&mask.val),
8978            simd: self,
8979        };
8980        Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
8981    }
8982    #[inline(always)]
8983    fn compress_merge_i64x4(
8984        self,
8985        values: i64x4<Self>,
8986        mask: mask64x4<Self>,
8987        merge: i64x4<Self>,
8988    ) -> i64x4<Self> {
8989        let mask = mask8x32 {
8990            val: crate::transmute::checked_transmute_copy(&mask.val),
8991            simd: self,
8992        };
8993        Bytes::from_bytes(self.compress_merge_u8x32(
8994            Bytes::to_bytes(values),
8995            mask,
8996            Bytes::to_bytes(merge),
8997        ))
8998    }
8999    #[inline(always)]
9000    fn expand_i64x4(self, values: i64x4<Self>, mask: mask64x4<Self>) -> i64x4<Self> {
9001        let mask = mask8x32 {
9002            val: crate::transmute::checked_transmute_copy(&mask.val),
9003            simd: self,
9004        };
9005        Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
9006    }
9007    #[inline(always)]
9008    fn expand_merge_i64x4(
9009        self,
9010        values: i64x4<Self>,
9011        mask: mask64x4<Self>,
9012        merge: i64x4<Self>,
9013    ) -> i64x4<Self> {
9014        let mask = mask8x32 {
9015            val: crate::transmute::checked_transmute_copy(&mask.val),
9016            simd: self,
9017        };
9018        Bytes::from_bytes(self.expand_merge_u8x32(
9019            Bytes::to_bytes(values),
9020            mask,
9021            Bytes::to_bytes(merge),
9022        ))
9023    }
9024    #[inline(always)]
9025    fn combine_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x8<Self> {
9026        i64x8 {
9027            val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
9028            simd: self,
9029        }
9030    }
9031    #[inline(always)]
9032    fn split_i64x4(self, a: i64x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
9033        (
9034            i64x2 {
9035                val: crate::support::Aligned128(a.val.0[0]),
9036                simd: self,
9037            },
9038            i64x2 {
9039                val: crate::support::Aligned128(a.val.0[1]),
9040                simd: self,
9041            },
9042        )
9043    }
9044    #[inline(always)]
9045    fn slide_u64x4<const SHIFT: usize>(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
9046        if SHIFT >= 4usize {
9047            return b;
9048        }
9049        let result = cross_block_alignr_128x2(
9050            self,
9051            Bytes::to_bytes(b).val.0,
9052            Bytes::to_bytes(a).val.0,
9053            SHIFT * 8usize,
9054        );
9055        Bytes::from_bytes(u8x32 {
9056            val: crate::support::Aligned256(result),
9057            simd: self,
9058        })
9059    }
9060    #[inline(always)]
9061    fn compress_u64x4(self, values: u64x4<Self>, mask: mask64x4<Self>) -> u64x4<Self> {
9062        let mask = mask8x32 {
9063            val: crate::transmute::checked_transmute_copy(&mask.val),
9064            simd: self,
9065        };
9066        Bytes::from_bytes(self.compress_u8x32(Bytes::to_bytes(values), mask))
9067    }
9068    #[inline(always)]
9069    fn compress_merge_u64x4(
9070        self,
9071        values: u64x4<Self>,
9072        mask: mask64x4<Self>,
9073        merge: u64x4<Self>,
9074    ) -> u64x4<Self> {
9075        let mask = mask8x32 {
9076            val: crate::transmute::checked_transmute_copy(&mask.val),
9077            simd: self,
9078        };
9079        Bytes::from_bytes(self.compress_merge_u8x32(
9080            Bytes::to_bytes(values),
9081            mask,
9082            Bytes::to_bytes(merge),
9083        ))
9084    }
9085    #[inline(always)]
9086    fn expand_u64x4(self, values: u64x4<Self>, mask: mask64x4<Self>) -> u64x4<Self> {
9087        let mask = mask8x32 {
9088            val: crate::transmute::checked_transmute_copy(&mask.val),
9089            simd: self,
9090        };
9091        Bytes::from_bytes(self.expand_u8x32(Bytes::to_bytes(values), mask))
9092    }
9093    #[inline(always)]
9094    fn expand_merge_u64x4(
9095        self,
9096        values: u64x4<Self>,
9097        mask: mask64x4<Self>,
9098        merge: u64x4<Self>,
9099    ) -> u64x4<Self> {
9100        let mask = mask8x32 {
9101            val: crate::transmute::checked_transmute_copy(&mask.val),
9102            simd: self,
9103        };
9104        Bytes::from_bytes(self.expand_merge_u8x32(
9105            Bytes::to_bytes(values),
9106            mask,
9107            Bytes::to_bytes(merge),
9108        ))
9109    }
9110    #[inline(always)]
9111    fn combine_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x8<Self> {
9112        u64x8 {
9113            val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
9114            simd: self,
9115        }
9116    }
9117    #[inline(always)]
9118    fn split_u64x4(self, a: u64x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
9119        (
9120            u64x2 {
9121                val: crate::support::Aligned128(a.val.0[0]),
9122                simd: self,
9123            },
9124            u64x2 {
9125                val: crate::support::Aligned128(a.val.0[1]),
9126                simd: self,
9127            },
9128        )
9129    }
9130    #[inline(always)]
9131    fn from_vector_mask64x4(self, vector: i64x4<Self>) -> mask64x4<Self> {
9132        mask64x4 {
9133            val: vector.val,
9134            simd: self,
9135        }
9136    }
9137    #[inline(always)]
9138    fn to_vector_mask64x4(self, a: mask64x4<Self>) -> i64x4<Self> {
9139        i64x4 {
9140            val: a.val,
9141            simd: self,
9142        }
9143    }
9144    #[inline(always)]
9145    fn set_mask64x4(self, a: &mut mask64x4<Self>, index: usize, value: bool) -> () {
9146        assert!(
9147            index < 4usize,
9148            "mask lane index {index} is out of bounds for {} lanes",
9149            4usize
9150        );
9151        let mut lanes: [i64; 4usize] = (*a).into();
9152        lanes[index] = if value { !0 } else { 0 };
9153        *a = lanes.simd_into(self);
9154    }
9155    #[inline(always)]
9156    fn rotate_elements_left_mask64x4<const OFFSET: usize>(
9157        self,
9158        a: mask64x4<Self>,
9159    ) -> mask64x4<Self> {
9160        let int = i64x4 {
9161            val: crate::transmute::checked_transmute_copy(&a.val),
9162            simd: self,
9163        };
9164        let rotated = int.rotate_elements_left::<OFFSET>();
9165        mask64x4 {
9166            val: crate::transmute::checked_transmute_copy(&rotated.val),
9167            simd: self,
9168        }
9169    }
9170    #[inline(always)]
9171    fn rotate_elements_right_mask64x4<const OFFSET: usize>(
9172        self,
9173        a: mask64x4<Self>,
9174    ) -> mask64x4<Self> {
9175        let int = i64x4 {
9176            val: crate::transmute::checked_transmute_copy(&a.val),
9177            simd: self,
9178        };
9179        let rotated = int.rotate_elements_right::<OFFSET>();
9180        mask64x4 {
9181            val: crate::transmute::checked_transmute_copy(&rotated.val),
9182            simd: self,
9183        }
9184    }
9185    #[inline(always)]
9186    fn combine_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x8<Self> {
9187        mask64x8 {
9188            val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
9189            simd: self,
9190        }
9191    }
9192    #[inline(always)]
9193    fn split_mask64x4(self, a: mask64x4<Self>) -> (mask64x2<Self>, mask64x2<Self>) {
9194        (
9195            mask64x2 {
9196                val: crate::support::Aligned128(a.val.0[0]),
9197                simd: self,
9198            },
9199            mask64x2 {
9200                val: crate::support::Aligned128(a.val.0[1]),
9201                simd: self,
9202            },
9203        )
9204    }
9205    #[inline(always)]
9206    fn slide_f32x16<const SHIFT: usize>(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
9207        if SHIFT >= 16usize {
9208            return b;
9209        }
9210        let result = cross_block_alignr_128x4(
9211            self,
9212            Bytes::to_bytes(b).val.0,
9213            Bytes::to_bytes(a).val.0,
9214            SHIFT * 4usize,
9215        );
9216        Bytes::from_bytes(u8x64 {
9217            val: crate::support::Aligned512(result),
9218            simd: self,
9219        })
9220    }
9221    #[inline(always)]
9222    fn compress_f32x16(self, values: f32x16<Self>, mask: mask32x16<Self>) -> f32x16<Self> {
9223        let mask = mask8x64 {
9224            val: crate::transmute::checked_transmute_copy(&mask.val),
9225            simd: self,
9226        };
9227        Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
9228    }
9229    #[inline(always)]
9230    fn compress_merge_f32x16(
9231        self,
9232        values: f32x16<Self>,
9233        mask: mask32x16<Self>,
9234        merge: f32x16<Self>,
9235    ) -> f32x16<Self> {
9236        let mask = mask8x64 {
9237            val: crate::transmute::checked_transmute_copy(&mask.val),
9238            simd: self,
9239        };
9240        Bytes::from_bytes(self.compress_merge_u8x64(
9241            Bytes::to_bytes(values),
9242            mask,
9243            Bytes::to_bytes(merge),
9244        ))
9245    }
9246    #[inline(always)]
9247    fn expand_f32x16(self, values: f32x16<Self>, mask: mask32x16<Self>) -> f32x16<Self> {
9248        let mask = mask8x64 {
9249            val: crate::transmute::checked_transmute_copy(&mask.val),
9250            simd: self,
9251        };
9252        Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
9253    }
9254    #[inline(always)]
9255    fn expand_merge_f32x16(
9256        self,
9257        values: f32x16<Self>,
9258        mask: mask32x16<Self>,
9259        merge: f32x16<Self>,
9260    ) -> f32x16<Self> {
9261        let mask = mask8x64 {
9262            val: crate::transmute::checked_transmute_copy(&mask.val),
9263            simd: self,
9264        };
9265        Bytes::from_bytes(self.expand_merge_u8x64(
9266            Bytes::to_bytes(values),
9267            mask,
9268            Bytes::to_bytes(merge),
9269        ))
9270    }
9271    #[inline(always)]
9272    fn simd_ne_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> mask32x16<Self> {
9273        let (a0, a1) = self.split_f32x16(a);
9274        let (b0, b1) = self.split_f32x16(b);
9275        self.combine_mask32x8(self.simd_ne_f32x8(a0, b0), self.simd_ne_f32x8(a1, b1))
9276    }
9277    #[inline(always)]
9278    fn is_nan_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
9279        a.simd_ne(a)
9280    }
9281    #[inline(always)]
9282    fn is_infinite_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
9283        a.abs().simd_eq(f32::INFINITY)
9284    }
9285    #[inline(always)]
9286    fn is_finite_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
9287        a.abs().simd_lt(f32::INFINITY)
9288    }
9289    #[inline(always)]
9290    fn is_subnormal_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
9291        let i: i32x16<Self> = a.bitcast();
9292        (i & 0x7F80_0000_i32).simd_eq(0) & (i & 0x007F_FFFF_i32).simd_ne(0)
9293    }
9294    #[inline(always)]
9295    fn is_normal_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
9296        let i: i32x16<Self> = a.bitcast();
9297        let exp = i & 0x7F80_0000_i32;
9298        exp.simd_ne(0) & exp.simd_ne(0x7F80_0000_i32)
9299    }
9300    #[inline(always)]
9301    fn is_sign_positive_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
9302        let i: i32x16<Self> = a.bitcast();
9303        (i & i32::MIN).simd_eq(0)
9304    }
9305    #[inline(always)]
9306    fn is_sign_negative_f32x16(self, a: f32x16<Self>) -> mask32x16<Self> {
9307        !a.is_sign_positive()
9308    }
9309    #[inline(always)]
9310    fn split_f32x16(self, a: f32x16<Self>) -> (f32x8<Self>, f32x8<Self>) {
9311        (
9312            f32x8 {
9313                val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
9314                simd: self,
9315            },
9316            f32x8 {
9317                val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
9318                simd: self,
9319            },
9320        )
9321    }
9322    #[inline(always)]
9323    fn slide_i8x64<const SHIFT: usize>(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
9324        if SHIFT >= 64usize {
9325            return b;
9326        }
9327        let result = cross_block_alignr_128x4(
9328            self,
9329            Bytes::to_bytes(b).val.0,
9330            Bytes::to_bytes(a).val.0,
9331            SHIFT,
9332        );
9333        Bytes::from_bytes(u8x64 {
9334            val: crate::support::Aligned512(result),
9335            simd: self,
9336        })
9337    }
9338    #[inline(always)]
9339    fn compress_i8x64(self, values: i8x64<Self>, mask: mask8x64<Self>) -> i8x64<Self> {
9340        let mask = mask8x64 {
9341            val: crate::transmute::checked_transmute_copy(&mask.val),
9342            simd: self,
9343        };
9344        Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
9345    }
9346    #[inline(always)]
9347    fn compress_merge_i8x64(
9348        self,
9349        values: i8x64<Self>,
9350        mask: mask8x64<Self>,
9351        merge: i8x64<Self>,
9352    ) -> i8x64<Self> {
9353        let mask = mask8x64 {
9354            val: crate::transmute::checked_transmute_copy(&mask.val),
9355            simd: self,
9356        };
9357        Bytes::from_bytes(self.compress_merge_u8x64(
9358            Bytes::to_bytes(values),
9359            mask,
9360            Bytes::to_bytes(merge),
9361        ))
9362    }
9363    #[inline(always)]
9364    fn expand_i8x64(self, values: i8x64<Self>, mask: mask8x64<Self>) -> i8x64<Self> {
9365        let mask = mask8x64 {
9366            val: crate::transmute::checked_transmute_copy(&mask.val),
9367            simd: self,
9368        };
9369        Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
9370    }
9371    #[inline(always)]
9372    fn expand_merge_i8x64(
9373        self,
9374        values: i8x64<Self>,
9375        mask: mask8x64<Self>,
9376        merge: i8x64<Self>,
9377    ) -> i8x64<Self> {
9378        let mask = mask8x64 {
9379            val: crate::transmute::checked_transmute_copy(&mask.val),
9380            simd: self,
9381        };
9382        Bytes::from_bytes(self.expand_merge_u8x64(
9383            Bytes::to_bytes(values),
9384            mask,
9385            Bytes::to_bytes(merge),
9386        ))
9387    }
9388    #[inline(always)]
9389    fn split_i8x64(self, a: i8x64<Self>) -> (i8x32<Self>, i8x32<Self>) {
9390        (
9391            i8x32 {
9392                val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
9393                simd: self,
9394            },
9395            i8x32 {
9396                val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
9397                simd: self,
9398            },
9399        )
9400    }
9401    #[inline(always)]
9402    fn slide_u8x64<const SHIFT: usize>(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
9403        if SHIFT >= 64usize {
9404            return b;
9405        }
9406        let result = cross_block_alignr_128x4(
9407            self,
9408            Bytes::to_bytes(b).val.0,
9409            Bytes::to_bytes(a).val.0,
9410            SHIFT,
9411        );
9412        Bytes::from_bytes(u8x64 {
9413            val: crate::support::Aligned512(result),
9414            simd: self,
9415        })
9416    }
9417    #[inline(always)]
9418    fn swizzle_dyn_u8x64(self, a: u8x64<Self>, indices: u8x64<Self>) -> u8x64<Self> {
9419        let bytes = Bytes::to_bytes(a);
9420        let mut output = [0u8; 64usize];
9421        for lane in 0..64usize {
9422            let index = indices[lane] as usize;
9423            output[lane] = bytes[index % 64usize];
9424        }
9425        let result: u8x64<Self> = output.simd_into(self);
9426        Bytes::from_bytes(result)
9427    }
9428    #[inline(always)]
9429    fn swizzle_dyn_precise_u8x64(self, a: u8x64<Self>, indices: u8x64<Self>) -> u8x64<Self> {
9430        let bytes = Bytes::to_bytes(a);
9431        let mut output = [0u8; 64usize];
9432        for lane in 0..64usize {
9433            let index = indices[lane] as usize;
9434            let value = bytes[index % 64usize];
9435            output[lane] = core::hint::select_unpredictable(index < 64usize, value, 0u8);
9436        }
9437        let result: u8x64<Self> = output.simd_into(self);
9438        Bytes::from_bytes(result)
9439    }
9440    #[inline(always)]
9441    fn concat_swizzle_dyn_u8x64(
9442        self,
9443        a: u8x64<Self>,
9444        b: u8x64<Self>,
9445        indices: u8x64<Self>,
9446    ) -> u8x64<Self> {
9447        let first_table = Bytes::to_bytes(a);
9448        let second_table = Bytes::to_bytes(b);
9449        let mut output = [0u8; 64usize];
9450        for lane in 0..64usize {
9451            let index = indices[lane] as usize % 128usize;
9452            output[lane] = if index < 64usize {
9453                first_table[index]
9454            } else {
9455                second_table[index - 64usize]
9456            };
9457        }
9458        let result: u8x64<Self> = output.simd_into(self);
9459        Bytes::from_bytes(result)
9460    }
9461    #[inline(always)]
9462    fn concat_swizzle_dyn_precise_u8x64(
9463        self,
9464        a: u8x64<Self>,
9465        b: u8x64<Self>,
9466        indices: u8x64<Self>,
9467    ) -> u8x64<Self> {
9468        let first_table = Bytes::to_bytes(a);
9469        let second_table = Bytes::to_bytes(b);
9470        let mut output = [0u8; 64usize];
9471        for lane in 0..64usize {
9472            let index = indices[lane] as usize;
9473            let table_index = index % 64usize;
9474            let value = if index < 64usize {
9475                first_table[table_index]
9476            } else {
9477                second_table[table_index]
9478            };
9479            output[lane] = if index < 128usize { value } else { 0 };
9480        }
9481        let result: u8x64<Self> = output.simd_into(self);
9482        Bytes::from_bytes(result)
9483    }
9484    #[inline(always)]
9485    fn compress_u8x64(self, values: u8x64<Self>, mask: mask8x64<Self>) -> u8x64<Self> {
9486        self.compress_merge_u8x64(values, mask, u8x64::splat(self, 0))
9487    }
9488    #[inline(always)]
9489    fn compress_merge_u8x64(
9490        self,
9491        values: u8x64<Self>,
9492        mask: mask8x64<Self>,
9493        merge: u8x64<Self>,
9494    ) -> u8x64<Self> {
9495        let mask = self.to_bitmask_mask8x64(mask);
9496        let mut merge_padded = [0u8; 64 + 1];
9497        merge_padded[..64].copy_from_slice(&*merge);
9498        let mut compacted = merge_padded;
9499        let mut output_lane = 0;
9500        for input_lane in 0..64 {
9501            compacted[output_lane] = values[input_lane];
9502            output_lane += ((mask >> input_lane) & 1) as usize;
9503        }
9504        compacted[output_lane] = merge_padded[output_lane];
9505        let mut result = merge;
9506        result.copy_from_slice(&compacted[..64]);
9507        result
9508    }
9509    #[inline(always)]
9510    fn expand_u8x64(self, values: u8x64<Self>, mask: mask8x64<Self>) -> u8x64<Self> {
9511        self.expand_merge_u8x64(values, mask, u8x64::splat(self, 0))
9512    }
9513    #[inline(always)]
9514    fn expand_merge_u8x64(
9515        self,
9516        values: u8x64<Self>,
9517        mask: mask8x64<Self>,
9518        merge: u8x64<Self>,
9519    ) -> u8x64<Self> {
9520        let mask = self.to_bitmask_mask8x64(mask);
9521        let mut result = merge;
9522        let mut input_lane = 0;
9523        for output_lane in 0..64 {
9524            let bit = ((mask >> output_lane) & 1) as u8;
9525            let keep = 0u8.wrapping_sub(bit);
9526            result[output_lane] = (values[input_lane] & keep) | (result[output_lane] & !keep);
9527            input_lane += usize::from(bit);
9528        }
9529        result
9530    }
9531    #[inline(always)]
9532    fn split_u8x64(self, a: u8x64<Self>) -> (u8x32<Self>, u8x32<Self>) {
9533        (
9534            u8x32 {
9535                val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
9536                simd: self,
9537            },
9538            u8x32 {
9539                val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
9540                simd: self,
9541            },
9542        )
9543    }
9544    #[inline(always)]
9545    fn from_vector_mask8x64(self, vector: i8x64<Self>) -> mask8x64<Self> {
9546        mask8x64 {
9547            val: vector.val,
9548            simd: self,
9549        }
9550    }
9551    #[inline(always)]
9552    fn to_vector_mask8x64(self, a: mask8x64<Self>) -> i8x64<Self> {
9553        i8x64 {
9554            val: a.val,
9555            simd: self,
9556        }
9557    }
9558    #[inline(always)]
9559    fn set_mask8x64(self, a: &mut mask8x64<Self>, index: usize, value: bool) -> () {
9560        assert!(
9561            index < 64usize,
9562            "mask lane index {index} is out of bounds for {} lanes",
9563            64usize
9564        );
9565        let mut lanes: [i8; 64usize] = (*a).into();
9566        lanes[index] = if value { !0 } else { 0 };
9567        *a = lanes.simd_into(self);
9568    }
9569    #[inline(always)]
9570    fn rotate_elements_left_mask8x64<const OFFSET: usize>(
9571        self,
9572        a: mask8x64<Self>,
9573    ) -> mask8x64<Self> {
9574        let int = i8x64 {
9575            val: crate::transmute::checked_transmute_copy(&a.val),
9576            simd: self,
9577        };
9578        let rotated = int.rotate_elements_left::<OFFSET>();
9579        mask8x64 {
9580            val: crate::transmute::checked_transmute_copy(&rotated.val),
9581            simd: self,
9582        }
9583    }
9584    #[inline(always)]
9585    fn rotate_elements_right_mask8x64<const OFFSET: usize>(
9586        self,
9587        a: mask8x64<Self>,
9588    ) -> mask8x64<Self> {
9589        let int = i8x64 {
9590            val: crate::transmute::checked_transmute_copy(&a.val),
9591            simd: self,
9592        };
9593        let rotated = int.rotate_elements_right::<OFFSET>();
9594        mask8x64 {
9595            val: crate::transmute::checked_transmute_copy(&rotated.val),
9596            simd: self,
9597        }
9598    }
9599    #[inline(always)]
9600    fn split_mask8x64(self, a: mask8x64<Self>) -> (mask8x32<Self>, mask8x32<Self>) {
9601        (
9602            mask8x32 {
9603                val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
9604                simd: self,
9605            },
9606            mask8x32 {
9607                val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
9608                simd: self,
9609            },
9610        )
9611    }
9612    #[inline(always)]
9613    fn slide_i16x32<const SHIFT: usize>(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
9614        if SHIFT >= 32usize {
9615            return b;
9616        }
9617        let result = cross_block_alignr_128x4(
9618            self,
9619            Bytes::to_bytes(b).val.0,
9620            Bytes::to_bytes(a).val.0,
9621            SHIFT * 2usize,
9622        );
9623        Bytes::from_bytes(u8x64 {
9624            val: crate::support::Aligned512(result),
9625            simd: self,
9626        })
9627    }
9628    #[inline(always)]
9629    fn compress_i16x32(self, values: i16x32<Self>, mask: mask16x32<Self>) -> i16x32<Self> {
9630        let mask = mask8x64 {
9631            val: crate::transmute::checked_transmute_copy(&mask.val),
9632            simd: self,
9633        };
9634        Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
9635    }
9636    #[inline(always)]
9637    fn compress_merge_i16x32(
9638        self,
9639        values: i16x32<Self>,
9640        mask: mask16x32<Self>,
9641        merge: i16x32<Self>,
9642    ) -> i16x32<Self> {
9643        let mask = mask8x64 {
9644            val: crate::transmute::checked_transmute_copy(&mask.val),
9645            simd: self,
9646        };
9647        Bytes::from_bytes(self.compress_merge_u8x64(
9648            Bytes::to_bytes(values),
9649            mask,
9650            Bytes::to_bytes(merge),
9651        ))
9652    }
9653    #[inline(always)]
9654    fn expand_i16x32(self, values: i16x32<Self>, mask: mask16x32<Self>) -> i16x32<Self> {
9655        let mask = mask8x64 {
9656            val: crate::transmute::checked_transmute_copy(&mask.val),
9657            simd: self,
9658        };
9659        Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
9660    }
9661    #[inline(always)]
9662    fn expand_merge_i16x32(
9663        self,
9664        values: i16x32<Self>,
9665        mask: mask16x32<Self>,
9666        merge: i16x32<Self>,
9667    ) -> i16x32<Self> {
9668        let mask = mask8x64 {
9669            val: crate::transmute::checked_transmute_copy(&mask.val),
9670            simd: self,
9671        };
9672        Bytes::from_bytes(self.expand_merge_u8x64(
9673            Bytes::to_bytes(values),
9674            mask,
9675            Bytes::to_bytes(merge),
9676        ))
9677    }
9678    #[inline(always)]
9679    fn split_i16x32(self, a: i16x32<Self>) -> (i16x16<Self>, i16x16<Self>) {
9680        (
9681            i16x16 {
9682                val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
9683                simd: self,
9684            },
9685            i16x16 {
9686                val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
9687                simd: self,
9688            },
9689        )
9690    }
9691    #[inline(always)]
9692    fn slide_u16x32<const SHIFT: usize>(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
9693        if SHIFT >= 32usize {
9694            return b;
9695        }
9696        let result = cross_block_alignr_128x4(
9697            self,
9698            Bytes::to_bytes(b).val.0,
9699            Bytes::to_bytes(a).val.0,
9700            SHIFT * 2usize,
9701        );
9702        Bytes::from_bytes(u8x64 {
9703            val: crate::support::Aligned512(result),
9704            simd: self,
9705        })
9706    }
9707    #[inline(always)]
9708    fn compress_u16x32(self, values: u16x32<Self>, mask: mask16x32<Self>) -> u16x32<Self> {
9709        let mask = mask8x64 {
9710            val: crate::transmute::checked_transmute_copy(&mask.val),
9711            simd: self,
9712        };
9713        Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
9714    }
9715    #[inline(always)]
9716    fn compress_merge_u16x32(
9717        self,
9718        values: u16x32<Self>,
9719        mask: mask16x32<Self>,
9720        merge: u16x32<Self>,
9721    ) -> u16x32<Self> {
9722        let mask = mask8x64 {
9723            val: crate::transmute::checked_transmute_copy(&mask.val),
9724            simd: self,
9725        };
9726        Bytes::from_bytes(self.compress_merge_u8x64(
9727            Bytes::to_bytes(values),
9728            mask,
9729            Bytes::to_bytes(merge),
9730        ))
9731    }
9732    #[inline(always)]
9733    fn expand_u16x32(self, values: u16x32<Self>, mask: mask16x32<Self>) -> u16x32<Self> {
9734        let mask = mask8x64 {
9735            val: crate::transmute::checked_transmute_copy(&mask.val),
9736            simd: self,
9737        };
9738        Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
9739    }
9740    #[inline(always)]
9741    fn expand_merge_u16x32(
9742        self,
9743        values: u16x32<Self>,
9744        mask: mask16x32<Self>,
9745        merge: u16x32<Self>,
9746    ) -> u16x32<Self> {
9747        let mask = mask8x64 {
9748            val: crate::transmute::checked_transmute_copy(&mask.val),
9749            simd: self,
9750        };
9751        Bytes::from_bytes(self.expand_merge_u8x64(
9752            Bytes::to_bytes(values),
9753            mask,
9754            Bytes::to_bytes(merge),
9755        ))
9756    }
9757    #[inline(always)]
9758    fn split_u16x32(self, a: u16x32<Self>) -> (u16x16<Self>, u16x16<Self>) {
9759        (
9760            u16x16 {
9761                val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
9762                simd: self,
9763            },
9764            u16x16 {
9765                val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
9766                simd: self,
9767            },
9768        )
9769    }
9770    #[inline(always)]
9771    fn to_bitmask_mask16x32(self, a: mask16x32<Self>) -> u64 {
9772        crate::kernel!(
9773            #[inline(always)]
9774            fn kernel(token: Sse2, a: mask16x32<Sse2>) -> u64 {
9775                {
9776                    let lo = _mm_packs_epi16(a.val.0[0], a.val.0[1]);
9777                    let hi = _mm_packs_epi16(a.val.0[2], a.val.0[3]);
9778                    let lo = _mm_movemask_epi8(lo) as u32 as u64;
9779                    let hi = _mm_movemask_epi8(hi) as u32 as u64;
9780                    lo | (hi << 16usize)
9781                }
9782            }
9783        );
9784        kernel(self, a)
9785    }
9786    #[inline(always)]
9787    fn from_vector_mask16x32(self, vector: i16x32<Self>) -> mask16x32<Self> {
9788        mask16x32 {
9789            val: vector.val,
9790            simd: self,
9791        }
9792    }
9793    #[inline(always)]
9794    fn to_vector_mask16x32(self, a: mask16x32<Self>) -> i16x32<Self> {
9795        i16x32 {
9796            val: a.val,
9797            simd: self,
9798        }
9799    }
9800    #[inline(always)]
9801    fn set_mask16x32(self, a: &mut mask16x32<Self>, index: usize, value: bool) -> () {
9802        assert!(
9803            index < 32usize,
9804            "mask lane index {index} is out of bounds for {} lanes",
9805            32usize
9806        );
9807        let mut lanes: [i16; 32usize] = (*a).into();
9808        lanes[index] = if value { !0 } else { 0 };
9809        *a = lanes.simd_into(self);
9810    }
9811    #[inline(always)]
9812    fn rotate_elements_left_mask16x32<const OFFSET: usize>(
9813        self,
9814        a: mask16x32<Self>,
9815    ) -> mask16x32<Self> {
9816        let int = i16x32 {
9817            val: crate::transmute::checked_transmute_copy(&a.val),
9818            simd: self,
9819        };
9820        let rotated = int.rotate_elements_left::<OFFSET>();
9821        mask16x32 {
9822            val: crate::transmute::checked_transmute_copy(&rotated.val),
9823            simd: self,
9824        }
9825    }
9826    #[inline(always)]
9827    fn rotate_elements_right_mask16x32<const OFFSET: usize>(
9828        self,
9829        a: mask16x32<Self>,
9830    ) -> mask16x32<Self> {
9831        let int = i16x32 {
9832            val: crate::transmute::checked_transmute_copy(&a.val),
9833            simd: self,
9834        };
9835        let rotated = int.rotate_elements_right::<OFFSET>();
9836        mask16x32 {
9837            val: crate::transmute::checked_transmute_copy(&rotated.val),
9838            simd: self,
9839        }
9840    }
9841    #[inline(always)]
9842    fn split_mask16x32(self, a: mask16x32<Self>) -> (mask16x16<Self>, mask16x16<Self>) {
9843        (
9844            mask16x16 {
9845                val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
9846                simd: self,
9847            },
9848            mask16x16 {
9849                val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
9850                simd: self,
9851            },
9852        )
9853    }
9854    #[inline(always)]
9855    fn slide_i32x16<const SHIFT: usize>(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
9856        if SHIFT >= 16usize {
9857            return b;
9858        }
9859        let result = cross_block_alignr_128x4(
9860            self,
9861            Bytes::to_bytes(b).val.0,
9862            Bytes::to_bytes(a).val.0,
9863            SHIFT * 4usize,
9864        );
9865        Bytes::from_bytes(u8x64 {
9866            val: crate::support::Aligned512(result),
9867            simd: self,
9868        })
9869    }
9870    #[inline(always)]
9871    fn compress_i32x16(self, values: i32x16<Self>, mask: mask32x16<Self>) -> i32x16<Self> {
9872        let mask = mask8x64 {
9873            val: crate::transmute::checked_transmute_copy(&mask.val),
9874            simd: self,
9875        };
9876        Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
9877    }
9878    #[inline(always)]
9879    fn compress_merge_i32x16(
9880        self,
9881        values: i32x16<Self>,
9882        mask: mask32x16<Self>,
9883        merge: i32x16<Self>,
9884    ) -> i32x16<Self> {
9885        let mask = mask8x64 {
9886            val: crate::transmute::checked_transmute_copy(&mask.val),
9887            simd: self,
9888        };
9889        Bytes::from_bytes(self.compress_merge_u8x64(
9890            Bytes::to_bytes(values),
9891            mask,
9892            Bytes::to_bytes(merge),
9893        ))
9894    }
9895    #[inline(always)]
9896    fn expand_i32x16(self, values: i32x16<Self>, mask: mask32x16<Self>) -> i32x16<Self> {
9897        let mask = mask8x64 {
9898            val: crate::transmute::checked_transmute_copy(&mask.val),
9899            simd: self,
9900        };
9901        Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
9902    }
9903    #[inline(always)]
9904    fn expand_merge_i32x16(
9905        self,
9906        values: i32x16<Self>,
9907        mask: mask32x16<Self>,
9908        merge: i32x16<Self>,
9909    ) -> i32x16<Self> {
9910        let mask = mask8x64 {
9911            val: crate::transmute::checked_transmute_copy(&mask.val),
9912            simd: self,
9913        };
9914        Bytes::from_bytes(self.expand_merge_u8x64(
9915            Bytes::to_bytes(values),
9916            mask,
9917            Bytes::to_bytes(merge),
9918        ))
9919    }
9920    #[inline(always)]
9921    fn split_i32x16(self, a: i32x16<Self>) -> (i32x8<Self>, i32x8<Self>) {
9922        (
9923            i32x8 {
9924                val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
9925                simd: self,
9926            },
9927            i32x8 {
9928                val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
9929                simd: self,
9930            },
9931        )
9932    }
9933    #[inline(always)]
9934    fn slide_u32x16<const SHIFT: usize>(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
9935        if SHIFT >= 16usize {
9936            return b;
9937        }
9938        let result = cross_block_alignr_128x4(
9939            self,
9940            Bytes::to_bytes(b).val.0,
9941            Bytes::to_bytes(a).val.0,
9942            SHIFT * 4usize,
9943        );
9944        Bytes::from_bytes(u8x64 {
9945            val: crate::support::Aligned512(result),
9946            simd: self,
9947        })
9948    }
9949    #[inline(always)]
9950    fn compress_u32x16(self, values: u32x16<Self>, mask: mask32x16<Self>) -> u32x16<Self> {
9951        let mask = mask8x64 {
9952            val: crate::transmute::checked_transmute_copy(&mask.val),
9953            simd: self,
9954        };
9955        Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
9956    }
9957    #[inline(always)]
9958    fn compress_merge_u32x16(
9959        self,
9960        values: u32x16<Self>,
9961        mask: mask32x16<Self>,
9962        merge: u32x16<Self>,
9963    ) -> u32x16<Self> {
9964        let mask = mask8x64 {
9965            val: crate::transmute::checked_transmute_copy(&mask.val),
9966            simd: self,
9967        };
9968        Bytes::from_bytes(self.compress_merge_u8x64(
9969            Bytes::to_bytes(values),
9970            mask,
9971            Bytes::to_bytes(merge),
9972        ))
9973    }
9974    #[inline(always)]
9975    fn expand_u32x16(self, values: u32x16<Self>, mask: mask32x16<Self>) -> u32x16<Self> {
9976        let mask = mask8x64 {
9977            val: crate::transmute::checked_transmute_copy(&mask.val),
9978            simd: self,
9979        };
9980        Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
9981    }
9982    #[inline(always)]
9983    fn expand_merge_u32x16(
9984        self,
9985        values: u32x16<Self>,
9986        mask: mask32x16<Self>,
9987        merge: u32x16<Self>,
9988    ) -> u32x16<Self> {
9989        let mask = mask8x64 {
9990            val: crate::transmute::checked_transmute_copy(&mask.val),
9991            simd: self,
9992        };
9993        Bytes::from_bytes(self.expand_merge_u8x64(
9994            Bytes::to_bytes(values),
9995            mask,
9996            Bytes::to_bytes(merge),
9997        ))
9998    }
9999    #[inline(always)]
10000    fn split_u32x16(self, a: u32x16<Self>) -> (u32x8<Self>, u32x8<Self>) {
10001        (
10002            u32x8 {
10003                val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
10004                simd: self,
10005            },
10006            u32x8 {
10007                val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
10008                simd: self,
10009            },
10010        )
10011    }
10012    #[inline(always)]
10013    fn from_vector_mask32x16(self, vector: i32x16<Self>) -> mask32x16<Self> {
10014        mask32x16 {
10015            val: vector.val,
10016            simd: self,
10017        }
10018    }
10019    #[inline(always)]
10020    fn to_vector_mask32x16(self, a: mask32x16<Self>) -> i32x16<Self> {
10021        i32x16 {
10022            val: a.val,
10023            simd: self,
10024        }
10025    }
10026    #[inline(always)]
10027    fn set_mask32x16(self, a: &mut mask32x16<Self>, index: usize, value: bool) -> () {
10028        assert!(
10029            index < 16usize,
10030            "mask lane index {index} is out of bounds for {} lanes",
10031            16usize
10032        );
10033        let mut lanes: [i32; 16usize] = (*a).into();
10034        lanes[index] = if value { !0 } else { 0 };
10035        *a = lanes.simd_into(self);
10036    }
10037    #[inline(always)]
10038    fn rotate_elements_left_mask32x16<const OFFSET: usize>(
10039        self,
10040        a: mask32x16<Self>,
10041    ) -> mask32x16<Self> {
10042        let int = i32x16 {
10043            val: crate::transmute::checked_transmute_copy(&a.val),
10044            simd: self,
10045        };
10046        let rotated = int.rotate_elements_left::<OFFSET>();
10047        mask32x16 {
10048            val: crate::transmute::checked_transmute_copy(&rotated.val),
10049            simd: self,
10050        }
10051    }
10052    #[inline(always)]
10053    fn rotate_elements_right_mask32x16<const OFFSET: usize>(
10054        self,
10055        a: mask32x16<Self>,
10056    ) -> mask32x16<Self> {
10057        let int = i32x16 {
10058            val: crate::transmute::checked_transmute_copy(&a.val),
10059            simd: self,
10060        };
10061        let rotated = int.rotate_elements_right::<OFFSET>();
10062        mask32x16 {
10063            val: crate::transmute::checked_transmute_copy(&rotated.val),
10064            simd: self,
10065        }
10066    }
10067    #[inline(always)]
10068    fn split_mask32x16(self, a: mask32x16<Self>) -> (mask32x8<Self>, mask32x8<Self>) {
10069        (
10070            mask32x8 {
10071                val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
10072                simd: self,
10073            },
10074            mask32x8 {
10075                val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
10076                simd: self,
10077            },
10078        )
10079    }
10080    #[inline(always)]
10081    fn slide_f64x8<const SHIFT: usize>(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
10082        if SHIFT >= 8usize {
10083            return b;
10084        }
10085        let result = cross_block_alignr_128x4(
10086            self,
10087            Bytes::to_bytes(b).val.0,
10088            Bytes::to_bytes(a).val.0,
10089            SHIFT * 8usize,
10090        );
10091        Bytes::from_bytes(u8x64 {
10092            val: crate::support::Aligned512(result),
10093            simd: self,
10094        })
10095    }
10096    #[inline(always)]
10097    fn compress_f64x8(self, values: f64x8<Self>, mask: mask64x8<Self>) -> f64x8<Self> {
10098        let mask = mask8x64 {
10099            val: crate::transmute::checked_transmute_copy(&mask.val),
10100            simd: self,
10101        };
10102        Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
10103    }
10104    #[inline(always)]
10105    fn compress_merge_f64x8(
10106        self,
10107        values: f64x8<Self>,
10108        mask: mask64x8<Self>,
10109        merge: f64x8<Self>,
10110    ) -> f64x8<Self> {
10111        let mask = mask8x64 {
10112            val: crate::transmute::checked_transmute_copy(&mask.val),
10113            simd: self,
10114        };
10115        Bytes::from_bytes(self.compress_merge_u8x64(
10116            Bytes::to_bytes(values),
10117            mask,
10118            Bytes::to_bytes(merge),
10119        ))
10120    }
10121    #[inline(always)]
10122    fn expand_f64x8(self, values: f64x8<Self>, mask: mask64x8<Self>) -> f64x8<Self> {
10123        let mask = mask8x64 {
10124            val: crate::transmute::checked_transmute_copy(&mask.val),
10125            simd: self,
10126        };
10127        Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
10128    }
10129    #[inline(always)]
10130    fn expand_merge_f64x8(
10131        self,
10132        values: f64x8<Self>,
10133        mask: mask64x8<Self>,
10134        merge: f64x8<Self>,
10135    ) -> f64x8<Self> {
10136        let mask = mask8x64 {
10137            val: crate::transmute::checked_transmute_copy(&mask.val),
10138            simd: self,
10139        };
10140        Bytes::from_bytes(self.expand_merge_u8x64(
10141            Bytes::to_bytes(values),
10142            mask,
10143            Bytes::to_bytes(merge),
10144        ))
10145    }
10146    #[inline(always)]
10147    fn simd_ne_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> mask64x8<Self> {
10148        let (a0, a1) = self.split_f64x8(a);
10149        let (b0, b1) = self.split_f64x8(b);
10150        self.combine_mask64x4(self.simd_ne_f64x4(a0, b0), self.simd_ne_f64x4(a1, b1))
10151    }
10152    #[inline(always)]
10153    fn is_nan_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
10154        a.simd_ne(a)
10155    }
10156    #[inline(always)]
10157    fn is_infinite_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
10158        a.abs().simd_eq(f64::INFINITY)
10159    }
10160    #[inline(always)]
10161    fn is_finite_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
10162        a.abs().simd_lt(f64::INFINITY)
10163    }
10164    #[inline(always)]
10165    fn is_subnormal_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
10166        let i: i64x8<Self> = a.bitcast();
10167        (i & 0x7FF0_0000_0000_0000_i64).simd_eq(0) & (i & 0x000F_FFFF_FFFF_FFFF_i64).simd_ne(0)
10168    }
10169    #[inline(always)]
10170    fn is_normal_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
10171        let i: i64x8<Self> = a.bitcast();
10172        let exp = i & 0x7FF0_0000_0000_0000_i64;
10173        exp.simd_ne(0) & exp.simd_ne(0x7FF0_0000_0000_0000_i64)
10174    }
10175    #[inline(always)]
10176    fn is_sign_positive_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
10177        let i: i64x8<Self> = a.bitcast();
10178        (i & i64::MIN).simd_eq(0)
10179    }
10180    #[inline(always)]
10181    fn is_sign_negative_f64x8(self, a: f64x8<Self>) -> mask64x8<Self> {
10182        !a.is_sign_positive()
10183    }
10184    #[inline(always)]
10185    fn split_f64x8(self, a: f64x8<Self>) -> (f64x4<Self>, f64x4<Self>) {
10186        (
10187            f64x4 {
10188                val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
10189                simd: self,
10190            },
10191            f64x4 {
10192                val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
10193                simd: self,
10194            },
10195        )
10196    }
10197    #[inline(always)]
10198    fn slide_i64x8<const SHIFT: usize>(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
10199        if SHIFT >= 8usize {
10200            return b;
10201        }
10202        let result = cross_block_alignr_128x4(
10203            self,
10204            Bytes::to_bytes(b).val.0,
10205            Bytes::to_bytes(a).val.0,
10206            SHIFT * 8usize,
10207        );
10208        Bytes::from_bytes(u8x64 {
10209            val: crate::support::Aligned512(result),
10210            simd: self,
10211        })
10212    }
10213    #[inline(always)]
10214    fn compress_i64x8(self, values: i64x8<Self>, mask: mask64x8<Self>) -> i64x8<Self> {
10215        let mask = mask8x64 {
10216            val: crate::transmute::checked_transmute_copy(&mask.val),
10217            simd: self,
10218        };
10219        Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
10220    }
10221    #[inline(always)]
10222    fn compress_merge_i64x8(
10223        self,
10224        values: i64x8<Self>,
10225        mask: mask64x8<Self>,
10226        merge: i64x8<Self>,
10227    ) -> i64x8<Self> {
10228        let mask = mask8x64 {
10229            val: crate::transmute::checked_transmute_copy(&mask.val),
10230            simd: self,
10231        };
10232        Bytes::from_bytes(self.compress_merge_u8x64(
10233            Bytes::to_bytes(values),
10234            mask,
10235            Bytes::to_bytes(merge),
10236        ))
10237    }
10238    #[inline(always)]
10239    fn expand_i64x8(self, values: i64x8<Self>, mask: mask64x8<Self>) -> i64x8<Self> {
10240        let mask = mask8x64 {
10241            val: crate::transmute::checked_transmute_copy(&mask.val),
10242            simd: self,
10243        };
10244        Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
10245    }
10246    #[inline(always)]
10247    fn expand_merge_i64x8(
10248        self,
10249        values: i64x8<Self>,
10250        mask: mask64x8<Self>,
10251        merge: i64x8<Self>,
10252    ) -> i64x8<Self> {
10253        let mask = mask8x64 {
10254            val: crate::transmute::checked_transmute_copy(&mask.val),
10255            simd: self,
10256        };
10257        Bytes::from_bytes(self.expand_merge_u8x64(
10258            Bytes::to_bytes(values),
10259            mask,
10260            Bytes::to_bytes(merge),
10261        ))
10262    }
10263    #[inline(always)]
10264    fn split_i64x8(self, a: i64x8<Self>) -> (i64x4<Self>, i64x4<Self>) {
10265        (
10266            i64x4 {
10267                val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
10268                simd: self,
10269            },
10270            i64x4 {
10271                val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
10272                simd: self,
10273            },
10274        )
10275    }
10276    #[inline(always)]
10277    fn slide_u64x8<const SHIFT: usize>(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
10278        if SHIFT >= 8usize {
10279            return b;
10280        }
10281        let result = cross_block_alignr_128x4(
10282            self,
10283            Bytes::to_bytes(b).val.0,
10284            Bytes::to_bytes(a).val.0,
10285            SHIFT * 8usize,
10286        );
10287        Bytes::from_bytes(u8x64 {
10288            val: crate::support::Aligned512(result),
10289            simd: self,
10290        })
10291    }
10292    #[inline(always)]
10293    fn compress_u64x8(self, values: u64x8<Self>, mask: mask64x8<Self>) -> u64x8<Self> {
10294        let mask = mask8x64 {
10295            val: crate::transmute::checked_transmute_copy(&mask.val),
10296            simd: self,
10297        };
10298        Bytes::from_bytes(self.compress_u8x64(Bytes::to_bytes(values), mask))
10299    }
10300    #[inline(always)]
10301    fn compress_merge_u64x8(
10302        self,
10303        values: u64x8<Self>,
10304        mask: mask64x8<Self>,
10305        merge: u64x8<Self>,
10306    ) -> u64x8<Self> {
10307        let mask = mask8x64 {
10308            val: crate::transmute::checked_transmute_copy(&mask.val),
10309            simd: self,
10310        };
10311        Bytes::from_bytes(self.compress_merge_u8x64(
10312            Bytes::to_bytes(values),
10313            mask,
10314            Bytes::to_bytes(merge),
10315        ))
10316    }
10317    #[inline(always)]
10318    fn expand_u64x8(self, values: u64x8<Self>, mask: mask64x8<Self>) -> u64x8<Self> {
10319        let mask = mask8x64 {
10320            val: crate::transmute::checked_transmute_copy(&mask.val),
10321            simd: self,
10322        };
10323        Bytes::from_bytes(self.expand_u8x64(Bytes::to_bytes(values), mask))
10324    }
10325    #[inline(always)]
10326    fn expand_merge_u64x8(
10327        self,
10328        values: u64x8<Self>,
10329        mask: mask64x8<Self>,
10330        merge: u64x8<Self>,
10331    ) -> u64x8<Self> {
10332        let mask = mask8x64 {
10333            val: crate::transmute::checked_transmute_copy(&mask.val),
10334            simd: self,
10335        };
10336        Bytes::from_bytes(self.expand_merge_u8x64(
10337            Bytes::to_bytes(values),
10338            mask,
10339            Bytes::to_bytes(merge),
10340        ))
10341    }
10342    #[inline(always)]
10343    fn split_u64x8(self, a: u64x8<Self>) -> (u64x4<Self>, u64x4<Self>) {
10344        (
10345            u64x4 {
10346                val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
10347                simd: self,
10348            },
10349            u64x4 {
10350                val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
10351                simd: self,
10352            },
10353        )
10354    }
10355    #[inline(always)]
10356    fn from_vector_mask64x8(self, vector: i64x8<Self>) -> mask64x8<Self> {
10357        mask64x8 {
10358            val: vector.val,
10359            simd: self,
10360        }
10361    }
10362    #[inline(always)]
10363    fn to_vector_mask64x8(self, a: mask64x8<Self>) -> i64x8<Self> {
10364        i64x8 {
10365            val: a.val,
10366            simd: self,
10367        }
10368    }
10369    #[inline(always)]
10370    fn set_mask64x8(self, a: &mut mask64x8<Self>, index: usize, value: bool) -> () {
10371        assert!(
10372            index < 8usize,
10373            "mask lane index {index} is out of bounds for {} lanes",
10374            8usize
10375        );
10376        let mut lanes: [i64; 8usize] = (*a).into();
10377        lanes[index] = if value { !0 } else { 0 };
10378        *a = lanes.simd_into(self);
10379    }
10380    #[inline(always)]
10381    fn rotate_elements_left_mask64x8<const OFFSET: usize>(
10382        self,
10383        a: mask64x8<Self>,
10384    ) -> mask64x8<Self> {
10385        let int = i64x8 {
10386            val: crate::transmute::checked_transmute_copy(&a.val),
10387            simd: self,
10388        };
10389        let rotated = int.rotate_elements_left::<OFFSET>();
10390        mask64x8 {
10391            val: crate::transmute::checked_transmute_copy(&rotated.val),
10392            simd: self,
10393        }
10394    }
10395    #[inline(always)]
10396    fn rotate_elements_right_mask64x8<const OFFSET: usize>(
10397        self,
10398        a: mask64x8<Self>,
10399    ) -> mask64x8<Self> {
10400        let int = i64x8 {
10401            val: crate::transmute::checked_transmute_copy(&a.val),
10402            simd: self,
10403        };
10404        let rotated = int.rotate_elements_right::<OFFSET>();
10405        mask64x8 {
10406            val: crate::transmute::checked_transmute_copy(&rotated.val),
10407            simd: self,
10408        }
10409    }
10410    #[inline(always)]
10411    fn split_mask64x8(self, a: mask64x8<Self>) -> (mask64x4<Self>, mask64x4<Self>) {
10412        (
10413            mask64x4 {
10414                val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
10415                simd: self,
10416            },
10417            mask64x4 {
10418                val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
10419                simd: self,
10420            },
10421        )
10422    }
10423}
10424impl<S: Simd> SimdFrom<__m128, S> for f32x4<S> {
10425    #[inline(always)]
10426    fn simd_from(simd: S, arch: __m128) -> Self {
10427        Self {
10428            val: crate::transmute::checked_transmute_copy(&arch),
10429            simd,
10430        }
10431    }
10432}
10433impl<S: Simd> From<f32x4<S>> for __m128 {
10434    #[inline(always)]
10435    fn from(value: f32x4<S>) -> Self {
10436        crate::transmute::checked_transmute_copy(&value.val)
10437    }
10438}
10439impl<S: Simd> SimdFrom<__m128i, S> for i8x16<S> {
10440    #[inline(always)]
10441    fn simd_from(simd: S, arch: __m128i) -> Self {
10442        Self {
10443            val: crate::transmute::checked_transmute_copy(&arch),
10444            simd,
10445        }
10446    }
10447}
10448impl<S: Simd> From<i8x16<S>> for __m128i {
10449    #[inline(always)]
10450    fn from(value: i8x16<S>) -> Self {
10451        crate::transmute::checked_transmute_copy(&value.val)
10452    }
10453}
10454impl<S: Simd> SimdFrom<__m128i, S> for u8x16<S> {
10455    #[inline(always)]
10456    fn simd_from(simd: S, arch: __m128i) -> Self {
10457        Self {
10458            val: crate::transmute::checked_transmute_copy(&arch),
10459            simd,
10460        }
10461    }
10462}
10463impl<S: Simd> From<u8x16<S>> for __m128i {
10464    #[inline(always)]
10465    fn from(value: u8x16<S>) -> Self {
10466        crate::transmute::checked_transmute_copy(&value.val)
10467    }
10468}
10469impl<S: Simd> SimdFrom<__m128i, S> for mask8x16<S> {
10470    #[inline(always)]
10471    fn simd_from(simd: S, arch: __m128i) -> Self {
10472        let lanes: [i8; 16usize] = crate::transmute::checked_transmute_copy(&arch);
10473        lanes.simd_into(simd)
10474    }
10475}
10476impl<S: Simd> From<mask8x16<S>> for __m128i {
10477    #[inline(always)]
10478    fn from(value: mask8x16<S>) -> Self {
10479        let lanes: [i8; 16usize] = value.into();
10480        crate::transmute::checked_transmute_copy(&lanes)
10481    }
10482}
10483impl<S: Simd> SimdFrom<__m128i, S> for i16x8<S> {
10484    #[inline(always)]
10485    fn simd_from(simd: S, arch: __m128i) -> Self {
10486        Self {
10487            val: crate::transmute::checked_transmute_copy(&arch),
10488            simd,
10489        }
10490    }
10491}
10492impl<S: Simd> From<i16x8<S>> for __m128i {
10493    #[inline(always)]
10494    fn from(value: i16x8<S>) -> Self {
10495        crate::transmute::checked_transmute_copy(&value.val)
10496    }
10497}
10498impl<S: Simd> SimdFrom<__m128i, S> for u16x8<S> {
10499    #[inline(always)]
10500    fn simd_from(simd: S, arch: __m128i) -> Self {
10501        Self {
10502            val: crate::transmute::checked_transmute_copy(&arch),
10503            simd,
10504        }
10505    }
10506}
10507impl<S: Simd> From<u16x8<S>> for __m128i {
10508    #[inline(always)]
10509    fn from(value: u16x8<S>) -> Self {
10510        crate::transmute::checked_transmute_copy(&value.val)
10511    }
10512}
10513impl<S: Simd> SimdFrom<__m128i, S> for mask16x8<S> {
10514    #[inline(always)]
10515    fn simd_from(simd: S, arch: __m128i) -> Self {
10516        let lanes: [i16; 8usize] = crate::transmute::checked_transmute_copy(&arch);
10517        lanes.simd_into(simd)
10518    }
10519}
10520impl<S: Simd> From<mask16x8<S>> for __m128i {
10521    #[inline(always)]
10522    fn from(value: mask16x8<S>) -> Self {
10523        let lanes: [i16; 8usize] = value.into();
10524        crate::transmute::checked_transmute_copy(&lanes)
10525    }
10526}
10527impl<S: Simd> SimdFrom<__m128i, S> for i32x4<S> {
10528    #[inline(always)]
10529    fn simd_from(simd: S, arch: __m128i) -> Self {
10530        Self {
10531            val: crate::transmute::checked_transmute_copy(&arch),
10532            simd,
10533        }
10534    }
10535}
10536impl<S: Simd> From<i32x4<S>> for __m128i {
10537    #[inline(always)]
10538    fn from(value: i32x4<S>) -> Self {
10539        crate::transmute::checked_transmute_copy(&value.val)
10540    }
10541}
10542impl<S: Simd> SimdFrom<__m128i, S> for u32x4<S> {
10543    #[inline(always)]
10544    fn simd_from(simd: S, arch: __m128i) -> Self {
10545        Self {
10546            val: crate::transmute::checked_transmute_copy(&arch),
10547            simd,
10548        }
10549    }
10550}
10551impl<S: Simd> From<u32x4<S>> for __m128i {
10552    #[inline(always)]
10553    fn from(value: u32x4<S>) -> Self {
10554        crate::transmute::checked_transmute_copy(&value.val)
10555    }
10556}
10557impl<S: Simd> SimdFrom<__m128i, S> for mask32x4<S> {
10558    #[inline(always)]
10559    fn simd_from(simd: S, arch: __m128i) -> Self {
10560        let lanes: [i32; 4usize] = crate::transmute::checked_transmute_copy(&arch);
10561        lanes.simd_into(simd)
10562    }
10563}
10564impl<S: Simd> From<mask32x4<S>> for __m128i {
10565    #[inline(always)]
10566    fn from(value: mask32x4<S>) -> Self {
10567        let lanes: [i32; 4usize] = value.into();
10568        crate::transmute::checked_transmute_copy(&lanes)
10569    }
10570}
10571impl<S: Simd> SimdFrom<__m128d, S> for f64x2<S> {
10572    #[inline(always)]
10573    fn simd_from(simd: S, arch: __m128d) -> Self {
10574        Self {
10575            val: crate::transmute::checked_transmute_copy(&arch),
10576            simd,
10577        }
10578    }
10579}
10580impl<S: Simd> From<f64x2<S>> for __m128d {
10581    #[inline(always)]
10582    fn from(value: f64x2<S>) -> Self {
10583        crate::transmute::checked_transmute_copy(&value.val)
10584    }
10585}
10586impl<S: Simd> SimdFrom<__m128i, S> for i64x2<S> {
10587    #[inline(always)]
10588    fn simd_from(simd: S, arch: __m128i) -> Self {
10589        Self {
10590            val: crate::transmute::checked_transmute_copy(&arch),
10591            simd,
10592        }
10593    }
10594}
10595impl<S: Simd> From<i64x2<S>> for __m128i {
10596    #[inline(always)]
10597    fn from(value: i64x2<S>) -> Self {
10598        crate::transmute::checked_transmute_copy(&value.val)
10599    }
10600}
10601impl<S: Simd> SimdFrom<__m128i, S> for u64x2<S> {
10602    #[inline(always)]
10603    fn simd_from(simd: S, arch: __m128i) -> Self {
10604        Self {
10605            val: crate::transmute::checked_transmute_copy(&arch),
10606            simd,
10607        }
10608    }
10609}
10610impl<S: Simd> From<u64x2<S>> for __m128i {
10611    #[inline(always)]
10612    fn from(value: u64x2<S>) -> Self {
10613        crate::transmute::checked_transmute_copy(&value.val)
10614    }
10615}
10616impl<S: Simd> SimdFrom<__m128i, S> for mask64x2<S> {
10617    #[inline(always)]
10618    fn simd_from(simd: S, arch: __m128i) -> Self {
10619        let lanes: [i64; 2usize] = crate::transmute::checked_transmute_copy(&arch);
10620        lanes.simd_into(simd)
10621    }
10622}
10623impl<S: Simd> From<mask64x2<S>> for __m128i {
10624    #[inline(always)]
10625    fn from(value: mask64x2<S>) -> Self {
10626        let lanes: [i64; 2usize] = value.into();
10627        crate::transmute::checked_transmute_copy(&lanes)
10628    }
10629}
10630crate::kernel!(
10631    #[doc = r" This is a version of the `alignr` intrinsic that takes a non-const shift argument. The shift is still"]
10632    #[doc = r" expected to be constant in practice, so the match statement will be optimized out. This exists because"]
10633    #[doc = r" Rust doesn't currently let you do math on const generics."]
10634    #[inline(always)]
10635    fn dyn_alignr_128(token: Sse2, a: __m128i, b: __m128i, shift: usize) -> __m128i {
10636        match shift {
10637            0usize => {
10638                let lo = _mm_srli_si128::<0i32>(b);
10639                let hi = _mm_slli_si128::<16i32>(a);
10640                _mm_or_si128(lo, hi)
10641            }
10642            1usize => {
10643                let lo = _mm_srli_si128::<1i32>(b);
10644                let hi = _mm_slli_si128::<15i32>(a);
10645                _mm_or_si128(lo, hi)
10646            }
10647            2usize => {
10648                let lo = _mm_srli_si128::<2i32>(b);
10649                let hi = _mm_slli_si128::<14i32>(a);
10650                _mm_or_si128(lo, hi)
10651            }
10652            3usize => {
10653                let lo = _mm_srli_si128::<3i32>(b);
10654                let hi = _mm_slli_si128::<13i32>(a);
10655                _mm_or_si128(lo, hi)
10656            }
10657            4usize => {
10658                let lo = _mm_srli_si128::<4i32>(b);
10659                let hi = _mm_slli_si128::<12i32>(a);
10660                _mm_or_si128(lo, hi)
10661            }
10662            5usize => {
10663                let lo = _mm_srli_si128::<5i32>(b);
10664                let hi = _mm_slli_si128::<11i32>(a);
10665                _mm_or_si128(lo, hi)
10666            }
10667            6usize => {
10668                let lo = _mm_srli_si128::<6i32>(b);
10669                let hi = _mm_slli_si128::<10i32>(a);
10670                _mm_or_si128(lo, hi)
10671            }
10672            7usize => {
10673                let lo = _mm_srli_si128::<7i32>(b);
10674                let hi = _mm_slli_si128::<9i32>(a);
10675                _mm_or_si128(lo, hi)
10676            }
10677            8usize => {
10678                let lo = _mm_srli_si128::<8i32>(b);
10679                let hi = _mm_slli_si128::<8i32>(a);
10680                _mm_or_si128(lo, hi)
10681            }
10682            9usize => {
10683                let lo = _mm_srli_si128::<9i32>(b);
10684                let hi = _mm_slli_si128::<7i32>(a);
10685                _mm_or_si128(lo, hi)
10686            }
10687            10usize => {
10688                let lo = _mm_srli_si128::<10i32>(b);
10689                let hi = _mm_slli_si128::<6i32>(a);
10690                _mm_or_si128(lo, hi)
10691            }
10692            11usize => {
10693                let lo = _mm_srli_si128::<11i32>(b);
10694                let hi = _mm_slli_si128::<5i32>(a);
10695                _mm_or_si128(lo, hi)
10696            }
10697            12usize => {
10698                let lo = _mm_srli_si128::<12i32>(b);
10699                let hi = _mm_slli_si128::<4i32>(a);
10700                _mm_or_si128(lo, hi)
10701            }
10702            13usize => {
10703                let lo = _mm_srli_si128::<13i32>(b);
10704                let hi = _mm_slli_si128::<3i32>(a);
10705                _mm_or_si128(lo, hi)
10706            }
10707            14usize => {
10708                let lo = _mm_srli_si128::<14i32>(b);
10709                let hi = _mm_slli_si128::<2i32>(a);
10710                _mm_or_si128(lo, hi)
10711            }
10712            15usize => {
10713                let lo = _mm_srli_si128::<15i32>(b);
10714                let hi = _mm_slli_si128::<1i32>(a);
10715                _mm_or_si128(lo, hi)
10716            }
10717            _ => unreachable!(),
10718        }
10719    }
10720);
10721crate::kernel!(
10722    #[doc = r" Concatenates `b` and `a` (each N blocks) and extracts N blocks starting at byte offset `shift_bytes`."]
10723    #[doc = r" Extracts from [b : a] (b in low bytes, a in high bytes), matching `alignr` semantics."]
10724    #[inline(always)]
10725    fn cross_block_alignr_128x2(
10726        token: Sse2,
10727        a: [__m128i; 2usize],
10728        b: [__m128i; 2usize],
10729        shift_bytes: usize,
10730    ) -> [__m128i; 2usize] {
10731        [
10732            {
10733                let [lo, hi] =
10734                    crate::support::cross_block_slide_blocks_at(&b, &a, 0usize, shift_bytes);
10735                dyn_alignr_128(token, hi, lo, shift_bytes % 16)
10736            },
10737            {
10738                let [lo, hi] =
10739                    crate::support::cross_block_slide_blocks_at(&b, &a, 1usize, shift_bytes);
10740                dyn_alignr_128(token, hi, lo, shift_bytes % 16)
10741            },
10742        ]
10743    }
10744);
10745crate::kernel!(
10746    #[doc = r" Concatenates `b` and `a` (each N blocks) and extracts N blocks starting at byte offset `shift_bytes`."]
10747    #[doc = r" Extracts from [b : a] (b in low bytes, a in high bytes), matching `alignr` semantics."]
10748    #[inline(always)]
10749    fn cross_block_alignr_128x4(
10750        token: Sse2,
10751        a: [__m128i; 4usize],
10752        b: [__m128i; 4usize],
10753        shift_bytes: usize,
10754    ) -> [__m128i; 4usize] {
10755        [
10756            {
10757                let [lo, hi] =
10758                    crate::support::cross_block_slide_blocks_at(&b, &a, 0usize, shift_bytes);
10759                dyn_alignr_128(token, hi, lo, shift_bytes % 16)
10760            },
10761            {
10762                let [lo, hi] =
10763                    crate::support::cross_block_slide_blocks_at(&b, &a, 1usize, shift_bytes);
10764                dyn_alignr_128(token, hi, lo, shift_bytes % 16)
10765            },
10766            {
10767                let [lo, hi] =
10768                    crate::support::cross_block_slide_blocks_at(&b, &a, 2usize, shift_bytes);
10769                dyn_alignr_128(token, hi, lo, shift_bytes % 16)
10770            },
10771            {
10772                let [lo, hi] =
10773                    crate::support::cross_block_slide_blocks_at(&b, &a, 3usize, shift_bytes);
10774                dyn_alignr_128(token, hi, lo, shift_bytes % 16)
10775            },
10776        ]
10777    }
10778);
fearless_simd/generated/sse2.rs

fearless_simd/generated/
sse2.rs
