use crate::{Level, arch_types::ArchTypes, prelude::*, seal::Seal};
use crate::{
f32x4, f32x8, f32x16, f64x2, f64x4, f64x8, i8x16, i8x32, i8x64, i16x8, i16x16, i16x32, i32x4,
i32x8, i32x16, i64x2, i64x4, i64x8, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16,
mask16x32, mask32x4, mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64,
u16x8, u16x16, u16x32, u32x4, u32x8, u32x16, u64x2, u64x4, u64x8,
};
#[cfg(target_arch = "x86")]
use core::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use core::arch::x86_64::*;
use core::ops::*;
#[doc = "A token for AVX2 intrinsics on `x86` and `x86_64`, representing the x86-64-v3 level."]
#[doc = "# Browsing the documentation"]
#[doc = "The method list on this struct is very verbose."]
#[doc = "A better way to browse the docs is by looking at vector types such as [`u32x4`], [`f32x4`] or [`mask32x4`]."]
#[doc = "They include all the operations listed here, and also provide some additional convenience methods."]
#[derive(Clone, Copy, Debug)]
pub struct Avx2 {
_private: (),
}
impl Avx2 {
#[doc = "Create a SIMD token proving that the x86-64-v3 features are available."]
#[doc = r""]
#[doc = r" Most users should safely obtain the token from [`Level::new`] instead of calling this function."]
#[doc = r""]
#[doc = r" This function can be called without an `unsafe` block from a function"]
#[doc = r" with all the required target features enabled via the `#[target_feature]` annotation."]
#[doc = r""]
#[doc = r" # Safety"]
#[doc = r""]
#[doc = "When invoking this function through an `unsafe` block, the caller must ensure that the current CPU supports `avx2`, `bmi1`, `bmi2`, `cmpxchg16b`, `f16c`, `fma`, `fxsr`, `lzcnt`, `movbe`, `popcnt`, `xsave`."]
#[inline]
#[target_feature(enable = "avx2,bmi1,bmi2,cmpxchg16b,f16c,fma,fxsr,lzcnt,movbe,popcnt,xsave")]
pub const fn assume_supported() -> Self {
Self { _private: () }
}
}
impl Seal for Avx2 {}
impl ArchTypes for Avx2 {
type f32x4 = crate::support::Aligned128<__m128>;
type i8x16 = crate::support::Aligned128<__m128i>;
type u8x16 = crate::support::Aligned128<__m128i>;
type mask8x16 = crate::support::Aligned128<__m128i>;
type i16x8 = crate::support::Aligned128<__m128i>;
type u16x8 = crate::support::Aligned128<__m128i>;
type mask16x8 = crate::support::Aligned128<__m128i>;
type i32x4 = crate::support::Aligned128<__m128i>;
type u32x4 = crate::support::Aligned128<__m128i>;
type mask32x4 = crate::support::Aligned128<__m128i>;
type f64x2 = crate::support::Aligned128<__m128d>;
type i64x2 = crate::support::Aligned128<__m128i>;
type u64x2 = crate::support::Aligned128<__m128i>;
type mask64x2 = crate::support::Aligned128<__m128i>;
type f32x8 = crate::support::Aligned256<__m256>;
type i8x32 = crate::support::Aligned256<__m256i>;
type u8x32 = crate::support::Aligned256<__m256i>;
type mask8x32 = crate::support::Aligned256<__m256i>;
type i16x16 = crate::support::Aligned256<__m256i>;
type u16x16 = crate::support::Aligned256<__m256i>;
type mask16x16 = crate::support::Aligned256<__m256i>;
type i32x8 = crate::support::Aligned256<__m256i>;
type u32x8 = crate::support::Aligned256<__m256i>;
type mask32x8 = crate::support::Aligned256<__m256i>;
type f64x4 = crate::support::Aligned256<__m256d>;
type i64x4 = crate::support::Aligned256<__m256i>;
type u64x4 = crate::support::Aligned256<__m256i>;
type mask64x4 = crate::support::Aligned256<__m256i>;
type f32x16 = crate::support::Aligned512<[__m256; 2usize]>;
type i8x64 = crate::support::Aligned512<[__m256i; 2usize]>;
type u8x64 = crate::support::Aligned512<[__m256i; 2usize]>;
type mask8x64 = crate::support::Aligned512<[__m256i; 2usize]>;
type i16x32 = crate::support::Aligned512<[__m256i; 2usize]>;
type u16x32 = crate::support::Aligned512<[__m256i; 2usize]>;
type mask16x32 = crate::support::Aligned512<[__m256i; 2usize]>;
type i32x16 = crate::support::Aligned512<[__m256i; 2usize]>;
type u32x16 = crate::support::Aligned512<[__m256i; 2usize]>;
type mask32x16 = crate::support::Aligned512<[__m256i; 2usize]>;
type f64x8 = crate::support::Aligned512<[__m256d; 2usize]>;
type i64x8 = crate::support::Aligned512<[__m256i; 2usize]>;
type u64x8 = crate::support::Aligned512<[__m256i; 2usize]>;
type mask64x8 = crate::support::Aligned512<[__m256i; 2usize]>;
}
impl Simd for Avx2 {
type f32s = f32x8<Self>;
type f64s = f64x4<Self>;
type u8s = u8x32<Self>;
type i8s = i8x32<Self>;
type u16s = u16x16<Self>;
type i16s = i16x16<Self>;
type u32s = u32x8<Self>;
type i32s = i32x8<Self>;
type u64s = u64x4<Self>;
type i64s = i64x4<Self>;
type mask8s = mask8x32<Self>;
type mask16s = mask16x16<Self>;
type mask32s = mask32x8<Self>;
type mask64s = mask64x4<Self>;
#[inline(always)]
fn level(self) -> Level {
Level::Avx2(self)
}
#[inline]
fn vectorize<F: FnOnce() -> R, R>(self, f: F) -> R {
#[target_feature(
enable = "avx2,bmi1,bmi2,cmpxchg16b,f16c,fma,fxsr,lzcnt,movbe,popcnt,xsave"
)]
fn vectorize_avx2<F: FnOnce() -> R, R>(f: F) -> R {
f()
}
unsafe { vectorize_avx2(f) }
}
#[inline(always)]
fn splat_f32x4(self, val: f32) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: f32) -> f32x4<Avx2> {
_mm_set1_ps(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_f32x4<const SHIFT: usize>(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = dyn_alignr_128(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn abs_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32x4<Avx2> {
_mm_andnot_ps(_mm_set1_ps(-0.0), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32x4<Avx2> {
_mm_xor_ps(a.into(), _mm_set1_ps(-0.0)).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn sqrt_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32x4<Avx2> {
_mm_sqrt_ps(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn approximate_recip_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32x4<Avx2> {
_mm_rcp_ps(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn add_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
_mm_add_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
_mm_sub_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
_mm_mul_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn div_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
_mm_div_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn copysign_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
let mask = _mm_set1_ps(-0.0);
_mm_or_ps(_mm_and_ps(mask, b.into()), _mm_andnot_ps(mask, a.into()))
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
_mm_max_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
_mm_min_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
let intermediate = _mm_max_ps(a.into(), b.into());
let b_is_nan = _mm_cmpunord_ps(b.into(), b.into());
_mm_blendv_ps(intermediate, a.into(), b_is_nan).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
let intermediate = _mm_min_ps(a.into(), b.into());
let b_is_nan = _mm_cmpunord_ps(b.into(), b.into());
_mm_blendv_ps(intermediate, a.into(), b_is_nan).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> mask32x4<Avx2> {
_mm_castps_si128(_mm_cmpeq_ps(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> mask32x4<Avx2> {
_mm_castps_si128(_mm_cmplt_ps(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> mask32x4<Avx2> {
_mm_castps_si128(_mm_cmple_ps(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
_mm_unpacklo_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
_mm_unpackhi_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
_mm_shuffle_ps::<0b10_00_10_00>(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x4<Avx2> {
_mm_shuffle_ps::<0b11_01_11_01>(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> (f32x4<Self>, f32x4<Self>) {
(self.zip_low_f32x4(a, b), self.zip_high_f32x4(a, b))
}
#[inline(always)]
fn deinterleave_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> (f32x4<Self>, f32x4<Self>) {
(self.unzip_low_f32x4(a, b), self.unzip_high_f32x4(a, b))
}
#[inline(always)]
fn mul_add_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>, c: f32x4<Avx2>) -> f32x4<Avx2> {
_mm_fmadd_ps(a.into(), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn mul_sub_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>, c: f32x4<Avx2>) -> f32x4<Avx2> {
_mm_fmsub_ps(a.into(), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn floor_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32x4<Avx2> {
_mm_round_ps::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn ceil_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32x4<Avx2> {
_mm_round_ps::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn round_ties_even_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32x4<Avx2> {
_mm_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn fract_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
a - self.trunc_f32x4(a)
}
#[inline(always)]
fn trunc_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>) -> f32x4<Avx2> {
_mm_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn select_f32x4(self, a: mask32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask32x4<Avx2>,
b: f32x4<Avx2>,
c: f32x4<Avx2>,
) -> f32x4<Avx2> {
_mm_blendv_ps(c.into(), b.into(), _mm_castsi128_ps(a.into())).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>, b: f32x4<Avx2>) -> f32x8<Avx2> {
_mm256_setr_m128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4<Self>; 4usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, src: &[f32; 16usize]) -> [f32x4<Avx2>; 4usize] {
let (chunks, []) = src.as_chunks::<4usize>() else {
unreachable!()
};
let v0: __m128 =
crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[0]);
let v1: __m128 =
crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[1]);
let v2: __m128 =
crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[2]);
let v3: __m128 =
crate::transmute::checked_transmute_copy::<[f32; 4usize], __m128>(&chunks[3]);
let tmp0 = _mm_unpacklo_ps(v0, v1);
let tmp1 = _mm_unpackhi_ps(v0, v1);
let tmp2 = _mm_unpacklo_ps(v2, v3);
let tmp3 = _mm_unpackhi_ps(v2, v3);
let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2)));
let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2)));
let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3)));
let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3)));
[
out0.simd_into(token),
out1.simd_into(token),
out2.simd_into(token),
out3.simd_into(token),
]
}
);
kernel(self, src)
}
#[inline(always)]
fn store_four_interleaved_f32x4(
self,
vectors: [f32x4<Self>; 4usize],
dest: &mut [f32; 16usize],
) -> () {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
vectors: [f32x4<Avx2>; 4usize],
dest: &mut [f32; 16usize],
) -> () {
let _ = token;
let v0: __m128 = vectors[0].into();
let v1: __m128 = vectors[1].into();
let v2: __m128 = vectors[2].into();
let v3: __m128 = vectors[3].into();
let tmp0 = _mm_unpacklo_ps(v0, v1);
let tmp1 = _mm_unpackhi_ps(v0, v1);
let tmp2 = _mm_unpacklo_ps(v2, v3);
let tmp3 = _mm_unpackhi_ps(v2, v3);
let out0 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2)));
let out1 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp0), _mm_castps_pd(tmp2)));
let out2 = _mm_castpd_ps(_mm_unpacklo_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3)));
let out3 = _mm_castpd_ps(_mm_unpackhi_pd(_mm_castps_pd(tmp1), _mm_castps_pd(tmp3)));
let (chunks, []) = dest.as_chunks_mut::<4usize>() else {
unreachable!()
};
crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>(
out0,
&mut chunks[0],
);
crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>(
out1,
&mut chunks[1],
);
crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>(
out2,
&mut chunks[2],
);
crate::transmute::checked_transmute_store::<__m128, [f32; 4usize]>(
out3,
&mut chunks[3],
);
}
);
kernel(self, vectors, dest);
}
#[inline(always)]
fn widen_f32x4(self, a: f32x4<Self>) -> (f64x2<Self>, f64x2<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>) -> (f64x2<Avx2>, f64x2<Avx2>) {
let raw = a.into();
(
_mm_cvtps_pd(raw).simd_into(token),
_mm_cvtps_pd(_mm_movehl_ps(raw, raw)).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_u32_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>) -> u32x4<Avx2> {
let mut converted = _mm_cvttps_epi32(a.into());
let in_range = _mm_cmplt_ps(a.into(), _mm_set1_ps(2147483648.0));
let all_in_range = _mm_movemask_ps(in_range) == 0b1111;
if !all_in_range {
let excess = _mm_sub_ps(a.into(), _mm_set1_ps(2147483648.0));
let excess_converted = _mm_cvttps_epi32(_mm_andnot_ps(in_range, excess));
converted = _mm_add_epi32(converted, excess_converted);
}
converted.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_u32_precise_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>) -> u32x4<Avx2> {
let a = _mm_max_ps(a.into(), _mm_setzero_ps());
let mut converted = _mm_cvttps_epi32(a);
let in_range = _mm_cmplt_ps(a, _mm_set1_ps(2147483648.0));
let all_in_range = _mm_movemask_ps(in_range) == 0b1111;
if !all_in_range {
let exceeds_unsigned_range =
_mm_castps_si128(_mm_cmplt_ps(_mm_set1_ps(4294967040.0), a));
let excess = _mm_sub_ps(a, _mm_set1_ps(2147483648.0));
let excess_converted = _mm_cvttps_epi32(_mm_andnot_ps(in_range, excess));
converted = _mm_add_epi32(converted, excess_converted);
converted = _mm_blendv_epi8(
converted,
_mm_set1_epi32(u32::MAX.cast_signed()),
exceeds_unsigned_range,
);
}
converted.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_i32_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>) -> i32x4<Avx2> {
_mm_cvttps_epi32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_i32_precise_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x4<Avx2>) -> i32x4<Avx2> {
let a = a.into();
let mut converted = _mm_cvttps_epi32(a);
let in_range = _mm_cmplt_ps(a, _mm_set1_ps(2147483648.0));
let all_in_range = _mm_movemask_ps(in_range) == 0b1111;
if !all_in_range {
converted = _mm_blendv_epi8(
_mm_set1_epi32(i32::MAX),
converted,
_mm_castps_si128(in_range),
);
let is_not_nan = _mm_castps_si128(_mm_cmpord_ps(a, a));
converted = _mm_and_si128(converted, is_not_nan);
}
converted.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_i8x16(self, val: i8) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: i8) -> i8x16<Avx2> {
_mm_set1_epi8(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_i8x16<const SHIFT: usize>(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = dyn_alignr_128(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
_mm_add_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
_mm_sub_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
let dst_even = _mm_mullo_epi16(a.into(), b.into());
let dst_odd =
_mm_mullo_epi16(_mm_srli_epi16::<8>(a.into()), _mm_srli_epi16::<8>(b.into()));
_mm_or_si128(
_mm_slli_epi16(dst_odd, 8),
_mm_and_si128(dst_even, _mm_set1_epi16(0xFF)),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
_mm_and_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
_mm_or_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
_mm_xor_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
a ^ !0
}
#[inline(always)]
fn shl_i8x16(self, a: i8x16<Self>, shift: u32) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x16<Avx2>, shift: u32) -> i8x16<Avx2> {
let val = a.into();
let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
let byte_mask = _mm_set1_epi8(mask_byte);
_mm_sll_epi16(_mm_and_si128(val, byte_mask), shift_count).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
[
i8::wrapping_shl(a[0usize], b[0usize] as u32),
i8::wrapping_shl(a[1usize], b[1usize] as u32),
i8::wrapping_shl(a[2usize], b[2usize] as u32),
i8::wrapping_shl(a[3usize], b[3usize] as u32),
i8::wrapping_shl(a[4usize], b[4usize] as u32),
i8::wrapping_shl(a[5usize], b[5usize] as u32),
i8::wrapping_shl(a[6usize], b[6usize] as u32),
i8::wrapping_shl(a[7usize], b[7usize] as u32),
i8::wrapping_shl(a[8usize], b[8usize] as u32),
i8::wrapping_shl(a[9usize], b[9usize] as u32),
i8::wrapping_shl(a[10usize], b[10usize] as u32),
i8::wrapping_shl(a[11usize], b[11usize] as u32),
i8::wrapping_shl(a[12usize], b[12usize] as u32),
i8::wrapping_shl(a[13usize], b[13usize] as u32),
i8::wrapping_shl(a[14usize], b[14usize] as u32),
i8::wrapping_shl(a[15usize], b[15usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn shr_i8x16(self, a: i8x16<Self>, shift: u32) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x16<Avx2>, shift: u32) -> i8x16<Avx2> {
let val = a.into();
let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
let byte_mask = _mm_set1_epi8(mask_byte);
let shifted = _mm_srl_epi16(val, shift_count);
let result = {
let sign = _mm_cmpgt_epi8(_mm_setzero_si128(), val);
_mm_or_si128(
_mm_and_si128(shifted, byte_mask),
_mm_andnot_si128(byte_mask, sign),
)
};
result.simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
[
i8::wrapping_shr(a[0usize], b[0usize] as u32),
i8::wrapping_shr(a[1usize], b[1usize] as u32),
i8::wrapping_shr(a[2usize], b[2usize] as u32),
i8::wrapping_shr(a[3usize], b[3usize] as u32),
i8::wrapping_shr(a[4usize], b[4usize] as u32),
i8::wrapping_shr(a[5usize], b[5usize] as u32),
i8::wrapping_shr(a[6usize], b[6usize] as u32),
i8::wrapping_shr(a[7usize], b[7usize] as u32),
i8::wrapping_shr(a[8usize], b[8usize] as u32),
i8::wrapping_shr(a[9usize], b[9usize] as u32),
i8::wrapping_shr(a[10usize], b[10usize] as u32),
i8::wrapping_shr(a[11usize], b[11usize] as u32),
i8::wrapping_shr(a[12usize], b[12usize] as u32),
i8::wrapping_shr(a[13usize], b[13usize] as u32),
i8::wrapping_shr(a[14usize], b[14usize] as u32),
i8::wrapping_shr(a[15usize], b[15usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn max_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
_mm_max_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
_mm_min_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> mask8x16<Avx2> {
_mm_cmpeq_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> mask8x16<Avx2> {
_mm_cmpgt_epi8(b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> mask8x16<Avx2> {
_mm_cmpeq_epi8(_mm_min_epi8(a.into(), b.into()), a.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
_mm_unpacklo_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
_mm_unpackhi_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
let t1 = _mm_shuffle_epi8(a.into(), mask);
let t2 = _mm_shuffle_epi8(b.into(), mask);
_mm_unpacklo_epi64(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x16<Avx2> {
let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
let t1 = _mm_shuffle_epi8(a.into(), mask);
let t2 = _mm_shuffle_epi8(b.into(), mask);
_mm_unpackhi_epi64(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i8x16<Self>, i8x16<Self>) {
(self.zip_low_i8x16(a, b), self.zip_high_i8x16(a, b))
}
#[inline(always)]
fn deinterleave_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i8x16<Self>, i8x16<Self>) {
(self.unzip_low_i8x16(a, b), self.unzip_high_i8x16(a, b))
}
#[inline(always)]
fn select_i8x16(self, a: mask8x16<Self>, b: i8x16<Self>, c: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask8x16<Avx2>,
b: i8x16<Avx2>,
c: i8x16<Avx2>,
) -> i8x16<Avx2> {
_mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x16<Avx2>, b: i8x16<Avx2>) -> i8x32<Avx2> {
_mm256_setr_m128i(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn neg_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x16<Avx2>) -> i8x16<Avx2> {
_mm_sub_epi8(_mm_setzero_si128(), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16<Self>; 4usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, src: &[i8; 64usize]) -> [i8x16<Avx2>; 4usize] {
let (chunks, []) = src.as_chunks::<16usize>() else {
unreachable!()
};
let v0: __m128i =
crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[0]);
let v1: __m128i =
crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[1]);
let v2: __m128i =
crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[2]);
let v3: __m128i =
crate::transmute::checked_transmute_copy::<[i8; 16usize], __m128i>(&chunks[3]);
let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
let v0 = _mm_shuffle_epi8(v0, mask);
let v1 = _mm_shuffle_epi8(v1, mask);
let v2 = _mm_shuffle_epi8(v2, mask);
let v3 = _mm_shuffle_epi8(v3, mask);
let tmp0 = _mm_unpacklo_epi32(v0, v1);
let tmp1 = _mm_unpackhi_epi32(v0, v1);
let tmp2 = _mm_unpacklo_epi32(v2, v3);
let tmp3 = _mm_unpackhi_epi32(v2, v3);
let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
[
out0.simd_into(token),
out1.simd_into(token),
out2.simd_into(token),
out3.simd_into(token),
]
}
);
kernel(self, src)
}
#[inline(always)]
fn store_four_interleaved_i8x16(
self,
vectors: [i8x16<Self>; 4usize],
dest: &mut [i8; 64usize],
) -> () {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, vectors: [i8x16<Avx2>; 4usize], dest: &mut [i8; 64usize]) -> () {
let _ = token;
let v0: __m128i = vectors[0].into();
let v1: __m128i = vectors[1].into();
let v2: __m128i = vectors[2].into();
let v3: __m128i = vectors[3].into();
let tmp0 = _mm_unpacklo_epi32(v0, v1);
let tmp1 = _mm_unpackhi_epi32(v0, v1);
let tmp2 = _mm_unpacklo_epi32(v2, v3);
let tmp3 = _mm_unpackhi_epi32(v2, v3);
let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
let out0 = _mm_shuffle_epi8(out0, mask);
let out1 = _mm_shuffle_epi8(out1, mask);
let out2 = _mm_shuffle_epi8(out2, mask);
let out3 = _mm_shuffle_epi8(out3, mask);
let (chunks, []) = dest.as_chunks_mut::<16usize>() else {
unreachable!()
};
crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>(
out0,
&mut chunks[0],
);
crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>(
out1,
&mut chunks[1],
);
crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>(
out2,
&mut chunks[2],
);
crate::transmute::checked_transmute_store::<__m128i, [i8; 16usize]>(
out3,
&mut chunks[3],
);
}
);
kernel(self, vectors, dest);
}
#[inline(always)]
fn widen_i8x16(self, a: i8x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x16<Avx2>) -> (i16x8<Avx2>, i16x8<Avx2>) {
let raw = a.into();
(
_mm_cvtepi8_epi16(raw).simd_into(token),
_mm_cvtepi8_epi16(_mm_srli_si128::<8>(raw)).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_u8x16(self, val: u8) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: u8) -> u8x16<Avx2> {
_mm_set1_epi8(val.cast_signed()).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_u8x16<const SHIFT: usize>(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = dyn_alignr_128(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, indices: u8x16<Avx2>) -> u8x16<Avx2> {
let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, indices.into());
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn swizzle_dyn_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, indices: u8x16<Avx2>) -> u8x16<Avx2> {
let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, indices.into());
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn swizzle_dyn_precise_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, indices: u8x16<Avx2>) -> u8x16<Avx2> {
let indices = indices.into();
let index_out_of_range = _mm_add_epi8(indices, _mm_set1_epi8(112));
let zeroing_indices = _mm_or_si128(indices, index_out_of_range);
let result = _mm_shuffle_epi8(Bytes::to_bytes(a).val.0, zeroing_indices);
let result_bytes = u8x16 {
val: crate::support::Aligned128(result),
simd: token,
};
Bytes::from_bytes(result_bytes)
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
_mm_add_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
_mm_sub_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
let dst_even = _mm_mullo_epi16(a.into(), b.into());
let dst_odd =
_mm_mullo_epi16(_mm_srli_epi16::<8>(a.into()), _mm_srli_epi16::<8>(b.into()));
_mm_or_si128(
_mm_slli_epi16(dst_odd, 8),
_mm_and_si128(dst_even, _mm_set1_epi16(0xFF)),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
_mm_and_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
_mm_or_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
_mm_xor_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
a ^ !0
}
#[inline(always)]
fn shl_u8x16(self, a: u8x16<Self>, shift: u32) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, shift: u32) -> u8x16<Avx2> {
let val = a.into();
let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
let byte_mask = _mm_set1_epi8(mask_byte);
_mm_sll_epi16(_mm_and_si128(val, byte_mask), shift_count).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
[
u8::wrapping_shl(a[0usize], b[0usize] as u32),
u8::wrapping_shl(a[1usize], b[1usize] as u32),
u8::wrapping_shl(a[2usize], b[2usize] as u32),
u8::wrapping_shl(a[3usize], b[3usize] as u32),
u8::wrapping_shl(a[4usize], b[4usize] as u32),
u8::wrapping_shl(a[5usize], b[5usize] as u32),
u8::wrapping_shl(a[6usize], b[6usize] as u32),
u8::wrapping_shl(a[7usize], b[7usize] as u32),
u8::wrapping_shl(a[8usize], b[8usize] as u32),
u8::wrapping_shl(a[9usize], b[9usize] as u32),
u8::wrapping_shl(a[10usize], b[10usize] as u32),
u8::wrapping_shl(a[11usize], b[11usize] as u32),
u8::wrapping_shl(a[12usize], b[12usize] as u32),
u8::wrapping_shl(a[13usize], b[13usize] as u32),
u8::wrapping_shl(a[14usize], b[14usize] as u32),
u8::wrapping_shl(a[15usize], b[15usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn shr_u8x16(self, a: u8x16<Self>, shift: u32) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, shift: u32) -> u8x16<Avx2> {
let val = a.into();
let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
let byte_mask = _mm_set1_epi8(mask_byte);
let shifted = _mm_srl_epi16(val, shift_count);
let result = { _mm_and_si128(shifted, byte_mask) };
result.simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
[
u8::wrapping_shr(a[0usize], b[0usize] as u32),
u8::wrapping_shr(a[1usize], b[1usize] as u32),
u8::wrapping_shr(a[2usize], b[2usize] as u32),
u8::wrapping_shr(a[3usize], b[3usize] as u32),
u8::wrapping_shr(a[4usize], b[4usize] as u32),
u8::wrapping_shr(a[5usize], b[5usize] as u32),
u8::wrapping_shr(a[6usize], b[6usize] as u32),
u8::wrapping_shr(a[7usize], b[7usize] as u32),
u8::wrapping_shr(a[8usize], b[8usize] as u32),
u8::wrapping_shr(a[9usize], b[9usize] as u32),
u8::wrapping_shr(a[10usize], b[10usize] as u32),
u8::wrapping_shr(a[11usize], b[11usize] as u32),
u8::wrapping_shr(a[12usize], b[12usize] as u32),
u8::wrapping_shr(a[13usize], b[13usize] as u32),
u8::wrapping_shr(a[14usize], b[14usize] as u32),
u8::wrapping_shr(a[15usize], b[15usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn max_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
_mm_max_epu8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
_mm_min_epu8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> mask8x16<Avx2> {
_mm_cmpeq_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> mask8x16<Avx2> {
{
let sign_bit = _mm_set1_epi8(0x80u8.cast_signed());
let lhs_signed = _mm_xor_si128(b.into(), sign_bit);
let rhs_signed = _mm_xor_si128(a.into(), sign_bit);
_mm_cmpgt_epi8(lhs_signed, rhs_signed)
}
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> mask8x16<Avx2> {
_mm_cmpeq_epi8(_mm_min_epu8(a.into(), b.into()), a.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
_mm_unpacklo_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
_mm_unpackhi_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
let t1 = _mm_shuffle_epi8(a.into(), mask);
let t2 = _mm_shuffle_epi8(b.into(), mask);
_mm_unpacklo_epi64(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x16<Avx2> {
let mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
let t1 = _mm_shuffle_epi8(a.into(), mask);
let t2 = _mm_shuffle_epi8(b.into(), mask);
_mm_unpackhi_epi64(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u8x16<Self>, u8x16<Self>) {
(self.zip_low_u8x16(a, b), self.zip_high_u8x16(a, b))
}
#[inline(always)]
fn deinterleave_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u8x16<Self>, u8x16<Self>) {
(self.unzip_low_u8x16(a, b), self.unzip_high_u8x16(a, b))
}
#[inline(always)]
fn select_u8x16(self, a: mask8x16<Self>, b: u8x16<Self>, c: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask8x16<Avx2>,
b: u8x16<Avx2>,
c: u8x16<Avx2>,
) -> u8x16<Avx2> {
_mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>, b: u8x16<Avx2>) -> u8x32<Avx2> {
_mm256_setr_m128i(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16<Self>; 4usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, src: &[u8; 64usize]) -> [u8x16<Avx2>; 4usize] {
let (chunks, []) = src.as_chunks::<16usize>() else {
unreachable!()
};
let v0: __m128i =
crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[0]);
let v1: __m128i =
crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[1]);
let v2: __m128i =
crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[2]);
let v3: __m128i =
crate::transmute::checked_transmute_copy::<[u8; 16usize], __m128i>(&chunks[3]);
let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
let v0 = _mm_shuffle_epi8(v0, mask);
let v1 = _mm_shuffle_epi8(v1, mask);
let v2 = _mm_shuffle_epi8(v2, mask);
let v3 = _mm_shuffle_epi8(v3, mask);
let tmp0 = _mm_unpacklo_epi32(v0, v1);
let tmp1 = _mm_unpackhi_epi32(v0, v1);
let tmp2 = _mm_unpacklo_epi32(v2, v3);
let tmp3 = _mm_unpackhi_epi32(v2, v3);
let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
[
out0.simd_into(token),
out1.simd_into(token),
out2.simd_into(token),
out3.simd_into(token),
]
}
);
kernel(self, src)
}
#[inline(always)]
fn store_four_interleaved_u8x16(
self,
vectors: [u8x16<Self>; 4usize],
dest: &mut [u8; 64usize],
) -> () {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, vectors: [u8x16<Avx2>; 4usize], dest: &mut [u8; 64usize]) -> () {
let _ = token;
let v0: __m128i = vectors[0].into();
let v1: __m128i = vectors[1].into();
let v2: __m128i = vectors[2].into();
let v3: __m128i = vectors[3].into();
let tmp0 = _mm_unpacklo_epi32(v0, v1);
let tmp1 = _mm_unpackhi_epi32(v0, v1);
let tmp2 = _mm_unpacklo_epi32(v2, v3);
let tmp3 = _mm_unpackhi_epi32(v2, v3);
let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
let mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
let out0 = _mm_shuffle_epi8(out0, mask);
let out1 = _mm_shuffle_epi8(out1, mask);
let out2 = _mm_shuffle_epi8(out2, mask);
let out3 = _mm_shuffle_epi8(out3, mask);
let (chunks, []) = dest.as_chunks_mut::<16usize>() else {
unreachable!()
};
crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>(
out0,
&mut chunks[0],
);
crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>(
out1,
&mut chunks[1],
);
crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>(
out2,
&mut chunks[2],
);
crate::transmute::checked_transmute_store::<__m128i, [u8; 16usize]>(
out3,
&mut chunks[3],
);
}
);
kernel(self, vectors, dest);
}
#[inline(always)]
fn widen_u8x16(self, a: u8x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x16<Avx2>) -> (u16x8<Avx2>, u16x8<Avx2>) {
let raw = a.into();
(
_mm_cvtepu8_epi16(raw).simd_into(token),
_mm_cvtepu8_epi16(_mm_srli_si128::<8>(raw)).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_mask8x16(self, val: bool) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: bool) -> mask8x16<Avx2> {
let val: i8 = if val { !0 } else { 0 };
_mm_set1_epi8(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn from_bitmask_mask8x16(self, bits: u64) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, bits: u64) -> mask8x16<Avx2> {
{
let bit_bytes = _mm_cvtsi32_si128(bits as i32);
let bit_bytes = _mm_shuffle_epi8(
bit_bytes,
_mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1),
);
let bit_mask =
_mm_setr_epi8(1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128);
_mm_cmpeq_epi8(_mm_and_si128(bit_bytes, bit_mask), bit_mask)
}
.simd_into(token)
}
);
kernel(self, bits)
}
#[inline(always)]
fn to_bitmask_mask8x16(self, a: mask8x16<Self>) -> u64 {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask8x16<Avx2>) -> u64 {
_mm_movemask_epi8(a.into()) as u32 as u64
}
);
kernel(self, a)
}
#[inline(always)]
fn set_mask8x16(self, a: &mut mask8x16<Self>, index: usize, value: bool) -> () {
assert!(
index < 16usize,
"mask lane index {index} is out of bounds for {} lanes",
16usize
);
let mut lanes: [i8; 16usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn and_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask8x16<Avx2>, b: mask8x16<Avx2>) -> mask8x16<Avx2> {
_mm_and_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask8x16<Avx2>, b: mask8x16<Avx2>) -> mask8x16<Avx2> {
_mm_or_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask8x16<Avx2>, b: mask8x16<Avx2>) -> mask8x16<Avx2> {
_mm_xor_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_mask8x16(self, a: mask8x16<Self>) -> mask8x16<Self> {
self.xor_mask8x16(a, self.splat_mask8x16(true))
}
#[inline(always)]
fn select_mask8x16(
self,
a: mask8x16<Self>,
b: mask8x16<Self>,
c: mask8x16<Self>,
) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask8x16<Avx2>,
b: mask8x16<Avx2>,
c: mask8x16<Avx2>,
) -> mask8x16<Avx2> {
_mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn simd_eq_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask8x16<Avx2>, b: mask8x16<Avx2>) -> mask8x16<Avx2> {
_mm_cmpeq_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn any_true_mask8x16(self, a: mask8x16<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask8x16<Avx2>) -> bool {
_mm_movemask_epi8(a.into()) as u32 != 0
}
);
kernel(self, a)
}
#[inline(always)]
fn all_true_mask8x16(self, a: mask8x16<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask8x16<Avx2>) -> bool {
_mm_movemask_epi8(a.into()) as u32 == 0xffff
}
);
kernel(self, a)
}
#[inline(always)]
fn any_false_mask8x16(self, a: mask8x16<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask8x16<Avx2>) -> bool {
_mm_movemask_epi8(a.into()) as u32 != 0xffff
}
);
kernel(self, a)
}
#[inline(always)]
fn all_false_mask8x16(self, a: mask8x16<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask8x16<Avx2>) -> bool {
_mm_movemask_epi8(a.into()) as u32 == 0
}
);
kernel(self, a)
}
#[inline(always)]
fn combine_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask8x16<Avx2>, b: mask8x16<Avx2>) -> mask8x32<Avx2> {
_mm256_setr_m128i(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn splat_i16x8(self, val: i16) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: i16) -> i16x8<Avx2> {
_mm_set1_epi16(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_i16x8<const SHIFT: usize>(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = dyn_alignr_128(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 2usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
_mm_add_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
_mm_sub_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
_mm_mullo_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
_mm_and_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
_mm_or_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
_mm_xor_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
a ^ !0
}
#[inline(always)]
fn shl_i16x8(self, a: i16x8<Self>, shift: u32) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>, shift: u32) -> i16x8<Avx2> {
_mm_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
[
i16::wrapping_shl(a[0usize], b[0usize] as u32),
i16::wrapping_shl(a[1usize], b[1usize] as u32),
i16::wrapping_shl(a[2usize], b[2usize] as u32),
i16::wrapping_shl(a[3usize], b[3usize] as u32),
i16::wrapping_shl(a[4usize], b[4usize] as u32),
i16::wrapping_shl(a[5usize], b[5usize] as u32),
i16::wrapping_shl(a[6usize], b[6usize] as u32),
i16::wrapping_shl(a[7usize], b[7usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn shr_i16x8(self, a: i16x8<Self>, shift: u32) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>, shift: u32) -> i16x8<Avx2> {
_mm_sra_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
[
i16::wrapping_shr(a[0usize], b[0usize] as u32),
i16::wrapping_shr(a[1usize], b[1usize] as u32),
i16::wrapping_shr(a[2usize], b[2usize] as u32),
i16::wrapping_shr(a[3usize], b[3usize] as u32),
i16::wrapping_shr(a[4usize], b[4usize] as u32),
i16::wrapping_shr(a[5usize], b[5usize] as u32),
i16::wrapping_shr(a[6usize], b[6usize] as u32),
i16::wrapping_shr(a[7usize], b[7usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn max_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
_mm_max_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
_mm_min_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> mask16x8<Avx2> {
_mm_cmpeq_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> mask16x8<Avx2> {
_mm_cmpgt_epi16(b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> mask16x8<Avx2> {
_mm_cmpeq_epi16(_mm_min_epi16(a.into(), b.into()), a.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
_mm_unpacklo_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
_mm_unpackhi_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
let t1 = _mm_shuffle_epi8(a.into(), mask);
let t2 = _mm_shuffle_epi8(b.into(), mask);
_mm_unpacklo_epi64(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x8<Avx2> {
let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
let t1 = _mm_shuffle_epi8(a.into(), mask);
let t2 = _mm_shuffle_epi8(b.into(), mask);
_mm_unpackhi_epi64(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i16x8<Self>, i16x8<Self>) {
(self.zip_low_i16x8(a, b), self.zip_high_i16x8(a, b))
}
#[inline(always)]
fn deinterleave_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i16x8<Self>, i16x8<Self>) {
(self.unzip_low_i16x8(a, b), self.unzip_high_i16x8(a, b))
}
#[inline(always)]
fn select_i16x8(self, a: mask16x8<Self>, b: i16x8<Self>, c: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask16x8<Avx2>,
b: i16x8<Avx2>,
c: i16x8<Avx2>,
) -> i16x8<Avx2> {
_mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i16x16<Avx2> {
_mm256_setr_m128i(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn neg_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>) -> i16x8<Avx2> {
_mm_sub_epi16(_mm_setzero_si128(), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8<Self>; 4usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, src: &[i16; 32usize]) -> [i16x8<Avx2>; 4usize] {
let (chunks, []) = src.as_chunks::<8usize>() else {
unreachable!()
};
let v0: __m128i =
crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[0]);
let v1: __m128i =
crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[1]);
let v2: __m128i =
crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[2]);
let v3: __m128i =
crate::transmute::checked_transmute_copy::<[i16; 8usize], __m128i>(&chunks[3]);
let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15);
let v0 = _mm_shuffle_epi8(v0, mask);
let v1 = _mm_shuffle_epi8(v1, mask);
let v2 = _mm_shuffle_epi8(v2, mask);
let v3 = _mm_shuffle_epi8(v3, mask);
let tmp0 = _mm_unpacklo_epi32(v0, v1);
let tmp1 = _mm_unpackhi_epi32(v0, v1);
let tmp2 = _mm_unpacklo_epi32(v2, v3);
let tmp3 = _mm_unpackhi_epi32(v2, v3);
let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
[
out0.simd_into(token),
out1.simd_into(token),
out2.simd_into(token),
out3.simd_into(token),
]
}
);
kernel(self, src)
}
#[inline(always)]
fn store_four_interleaved_i16x8(
self,
vectors: [i16x8<Self>; 4usize],
dest: &mut [i16; 32usize],
) -> () {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
vectors: [i16x8<Avx2>; 4usize],
dest: &mut [i16; 32usize],
) -> () {
let _ = token;
let v0: __m128i = vectors[0].into();
let v1: __m128i = vectors[1].into();
let v2: __m128i = vectors[2].into();
let v3: __m128i = vectors[3].into();
let tmp0 = _mm_unpacklo_epi32(v0, v1);
let tmp1 = _mm_unpackhi_epi32(v0, v1);
let tmp2 = _mm_unpacklo_epi32(v2, v3);
let tmp3 = _mm_unpackhi_epi32(v2, v3);
let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
let out0 = _mm_shuffle_epi8(out0, mask);
let out1 = _mm_shuffle_epi8(out1, mask);
let out2 = _mm_shuffle_epi8(out2, mask);
let out3 = _mm_shuffle_epi8(out3, mask);
let (chunks, []) = dest.as_chunks_mut::<8usize>() else {
unreachable!()
};
crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>(
out0,
&mut chunks[0],
);
crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>(
out1,
&mut chunks[1],
);
crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>(
out2,
&mut chunks[2],
);
crate::transmute::checked_transmute_store::<__m128i, [i16; 8usize]>(
out3,
&mut chunks[3],
);
}
);
kernel(self, vectors, dest);
}
#[inline(always)]
fn widen_i16x8(self, a: i16x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>) -> (i32x4<Avx2>, i32x4<Avx2>) {
let raw = a.into();
(
_mm_cvtepi16_epi32(raw).simd_into(token),
_mm_cvtepi16_epi32(_mm_srli_si128::<8>(raw)).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i8x16<Avx2> {
let mask = _mm_set1_epi16(0xff);
_mm_packus_epi16(_mm_and_si128(a.into(), mask), _mm_and_si128(b.into(), mask))
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x8<Avx2>, b: i16x8<Avx2>) -> i8x16<Avx2> {
_mm_packs_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn relaxed_narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= i8::MIN as i16 && value <= i8::MAX as i16 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.saturating_narrow_i16x8(a, b)
}
#[inline(always)]
fn splat_u16x8(self, val: u16) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: u16) -> u16x8<Avx2> {
_mm_set1_epi16(val.cast_signed()).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_u16x8<const SHIFT: usize>(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = dyn_alignr_128(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 2usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
_mm_add_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
_mm_sub_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
_mm_mullo_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
_mm_and_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
_mm_or_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
_mm_xor_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
a ^ !0
}
#[inline(always)]
fn shl_u16x8(self, a: u16x8<Self>, shift: u32) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>, shift: u32) -> u16x8<Avx2> {
_mm_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
[
u16::wrapping_shl(a[0usize], b[0usize] as u32),
u16::wrapping_shl(a[1usize], b[1usize] as u32),
u16::wrapping_shl(a[2usize], b[2usize] as u32),
u16::wrapping_shl(a[3usize], b[3usize] as u32),
u16::wrapping_shl(a[4usize], b[4usize] as u32),
u16::wrapping_shl(a[5usize], b[5usize] as u32),
u16::wrapping_shl(a[6usize], b[6usize] as u32),
u16::wrapping_shl(a[7usize], b[7usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn shr_u16x8(self, a: u16x8<Self>, shift: u32) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>, shift: u32) -> u16x8<Avx2> {
_mm_srl_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
[
u16::wrapping_shr(a[0usize], b[0usize] as u32),
u16::wrapping_shr(a[1usize], b[1usize] as u32),
u16::wrapping_shr(a[2usize], b[2usize] as u32),
u16::wrapping_shr(a[3usize], b[3usize] as u32),
u16::wrapping_shr(a[4usize], b[4usize] as u32),
u16::wrapping_shr(a[5usize], b[5usize] as u32),
u16::wrapping_shr(a[6usize], b[6usize] as u32),
u16::wrapping_shr(a[7usize], b[7usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn max_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
_mm_max_epu16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
_mm_min_epu16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> mask16x8<Avx2> {
_mm_cmpeq_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> mask16x8<Avx2> {
{
let sign_bit = _mm_set1_epi16(0x8000u16.cast_signed());
let lhs_signed = _mm_xor_si128(b.into(), sign_bit);
let rhs_signed = _mm_xor_si128(a.into(), sign_bit);
_mm_cmpgt_epi16(lhs_signed, rhs_signed)
}
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> mask16x8<Avx2> {
_mm_cmpeq_epi16(_mm_min_epu16(a.into(), b.into()), a.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
_mm_unpacklo_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
_mm_unpackhi_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
let t1 = _mm_shuffle_epi8(a.into(), mask);
let t2 = _mm_shuffle_epi8(b.into(), mask);
_mm_unpacklo_epi64(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x8<Avx2> {
let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
let t1 = _mm_shuffle_epi8(a.into(), mask);
let t2 = _mm_shuffle_epi8(b.into(), mask);
_mm_unpackhi_epi64(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u16x8<Self>, u16x8<Self>) {
(self.zip_low_u16x8(a, b), self.zip_high_u16x8(a, b))
}
#[inline(always)]
fn deinterleave_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u16x8<Self>, u16x8<Self>) {
(self.unzip_low_u16x8(a, b), self.unzip_high_u16x8(a, b))
}
#[inline(always)]
fn select_u16x8(self, a: mask16x8<Self>, b: u16x8<Self>, c: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask16x8<Avx2>,
b: u16x8<Avx2>,
c: u16x8<Avx2>,
) -> u16x8<Avx2> {
_mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u16x16<Avx2> {
_mm256_setr_m128i(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8<Self>; 4usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, src: &[u16; 32usize]) -> [u16x8<Avx2>; 4usize] {
let (chunks, []) = src.as_chunks::<8usize>() else {
unreachable!()
};
let v0: __m128i =
crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[0]);
let v1: __m128i =
crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[1]);
let v2: __m128i =
crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[2]);
let v3: __m128i =
crate::transmute::checked_transmute_copy::<[u16; 8usize], __m128i>(&chunks[3]);
let mask = _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15);
let v0 = _mm_shuffle_epi8(v0, mask);
let v1 = _mm_shuffle_epi8(v1, mask);
let v2 = _mm_shuffle_epi8(v2, mask);
let v3 = _mm_shuffle_epi8(v3, mask);
let tmp0 = _mm_unpacklo_epi32(v0, v1);
let tmp1 = _mm_unpackhi_epi32(v0, v1);
let tmp2 = _mm_unpacklo_epi32(v2, v3);
let tmp3 = _mm_unpackhi_epi32(v2, v3);
let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
[
out0.simd_into(token),
out1.simd_into(token),
out2.simd_into(token),
out3.simd_into(token),
]
}
);
kernel(self, src)
}
#[inline(always)]
fn store_four_interleaved_u16x8(
self,
vectors: [u16x8<Self>; 4usize],
dest: &mut [u16; 32usize],
) -> () {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
vectors: [u16x8<Avx2>; 4usize],
dest: &mut [u16; 32usize],
) -> () {
let _ = token;
let v0: __m128i = vectors[0].into();
let v1: __m128i = vectors[1].into();
let v2: __m128i = vectors[2].into();
let v3: __m128i = vectors[3].into();
let tmp0 = _mm_unpacklo_epi32(v0, v1);
let tmp1 = _mm_unpackhi_epi32(v0, v1);
let tmp2 = _mm_unpacklo_epi32(v2, v3);
let tmp3 = _mm_unpackhi_epi32(v2, v3);
let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
let mask = _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
let out0 = _mm_shuffle_epi8(out0, mask);
let out1 = _mm_shuffle_epi8(out1, mask);
let out2 = _mm_shuffle_epi8(out2, mask);
let out3 = _mm_shuffle_epi8(out3, mask);
let (chunks, []) = dest.as_chunks_mut::<8usize>() else {
unreachable!()
};
crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>(
out0,
&mut chunks[0],
);
crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>(
out1,
&mut chunks[1],
);
crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>(
out2,
&mut chunks[2],
);
crate::transmute::checked_transmute_store::<__m128i, [u16; 8usize]>(
out3,
&mut chunks[3],
);
}
);
kernel(self, vectors, dest);
}
#[inline(always)]
fn widen_u16x8(self, a: u16x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>) -> (u32x4<Avx2>, u32x4<Avx2>) {
let raw = a.into();
(
_mm_cvtepu16_epi32(raw).simd_into(token),
_mm_cvtepu16_epi32(_mm_srli_si128::<8>(raw)).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u8x16<Avx2> {
let mask = _mm_set1_epi16(0xff);
_mm_packus_epi16(_mm_and_si128(a.into(), mask), _mm_and_si128(b.into(), mask))
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x8<Avx2>, b: u16x8<Avx2>) -> u8x16<Avx2> {
let max = _mm_set1_epi16(u8::MAX as i16);
let a = a.into();
let b = b.into();
let a = _mm_sub_epi16(a, _mm_subs_epu16(a, max));
let b = _mm_sub_epi16(b, _mm_subs_epu16(b, max));
_mm_packus_epi16(a, b).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn relaxed_narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= u8::MIN as u16 && value <= u8::MAX as u16 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.narrow_u16x8(a, b)
}
#[inline(always)]
fn splat_mask16x8(self, val: bool) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: bool) -> mask16x8<Avx2> {
let val: i16 = if val { !0 } else { 0 };
_mm_set1_epi16(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn from_bitmask_mask16x8(self, bits: u64) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, bits: u64) -> mask16x8<Avx2> {
{
let bit_lanes = _mm_set1_epi16(bits as i16);
let bit_mask = _mm_setr_epi16(1, 2, 4, 8, 16, 32, 64, 128);
_mm_cmpeq_epi16(_mm_and_si128(bit_lanes, bit_mask), bit_mask)
}
.simd_into(token)
}
);
kernel(self, bits)
}
#[inline(always)]
fn to_bitmask_mask16x8(self, a: mask16x8<Self>) -> u64 {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x8<Avx2>) -> u64 {
{
let packed = _mm_packs_epi16(a.into(), a.into());
_mm_movemask_epi8(packed) as u8 as u64
}
}
);
kernel(self, a)
}
#[inline(always)]
fn set_mask16x8(self, a: &mut mask16x8<Self>, index: usize, value: bool) -> () {
assert!(
index < 8usize,
"mask lane index {index} is out of bounds for {} lanes",
8usize
);
let mut lanes: [i16; 8usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn and_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x8<Avx2>, b: mask16x8<Avx2>) -> mask16x8<Avx2> {
_mm_and_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x8<Avx2>, b: mask16x8<Avx2>) -> mask16x8<Avx2> {
_mm_or_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x8<Avx2>, b: mask16x8<Avx2>) -> mask16x8<Avx2> {
_mm_xor_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_mask16x8(self, a: mask16x8<Self>) -> mask16x8<Self> {
self.xor_mask16x8(a, self.splat_mask16x8(true))
}
#[inline(always)]
fn select_mask16x8(
self,
a: mask16x8<Self>,
b: mask16x8<Self>,
c: mask16x8<Self>,
) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask16x8<Avx2>,
b: mask16x8<Avx2>,
c: mask16x8<Avx2>,
) -> mask16x8<Avx2> {
_mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn simd_eq_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x8<Avx2>, b: mask16x8<Avx2>) -> mask16x8<Avx2> {
_mm_cmpeq_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn any_true_mask16x8(self, a: mask16x8<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x8<Avx2>) -> bool {
_mm_movemask_epi8(a.into()) as u32 != 0
}
);
kernel(self, a)
}
#[inline(always)]
fn all_true_mask16x8(self, a: mask16x8<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x8<Avx2>) -> bool {
_mm_movemask_epi8(a.into()) as u32 == 0xffff
}
);
kernel(self, a)
}
#[inline(always)]
fn any_false_mask16x8(self, a: mask16x8<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x8<Avx2>) -> bool {
_mm_movemask_epi8(a.into()) as u32 != 0xffff
}
);
kernel(self, a)
}
#[inline(always)]
fn all_false_mask16x8(self, a: mask16x8<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x8<Avx2>) -> bool {
_mm_movemask_epi8(a.into()) as u32 == 0
}
);
kernel(self, a)
}
#[inline(always)]
fn combine_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x8<Avx2>, b: mask16x8<Avx2>) -> mask16x16<Avx2> {
_mm256_setr_m128i(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn splat_i32x4(self, val: i32) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: i32) -> i32x4<Avx2> {
_mm_set1_epi32(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_i32x4<const SHIFT: usize>(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = dyn_alignr_128(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
_mm_add_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
_mm_sub_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
_mm_mullo_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
_mm_and_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
_mm_or_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
_mm_xor_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
a ^ !0
}
#[inline(always)]
fn shl_i32x4(self, a: i32x4<Self>, shift: u32) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, shift: u32) -> i32x4<Avx2> {
_mm_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
_mm_sllv_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_i32x4(self, a: i32x4<Self>, shift: u32) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, shift: u32) -> i32x4<Avx2> {
_mm_sra_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
_mm_srav_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
_mm_max_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
_mm_min_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> mask32x4<Avx2> {
_mm_cmpeq_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> mask32x4<Avx2> {
_mm_cmpgt_epi32(b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> mask32x4<Avx2> {
_mm_cmpeq_epi32(_mm_min_epi32(a.into(), b.into()), a.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
_mm_unpacklo_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
_mm_unpackhi_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
let t1 = _mm_shuffle_epi32::<0b11_01_10_00>(a.into());
let t2 = _mm_shuffle_epi32::<0b11_01_10_00>(b.into());
_mm_unpacklo_epi64(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x4<Avx2> {
let t1 = _mm_shuffle_epi32::<0b11_01_10_00>(a.into());
let t2 = _mm_shuffle_epi32::<0b11_01_10_00>(b.into());
_mm_unpackhi_epi64(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i32x4<Self>, i32x4<Self>) {
(self.zip_low_i32x4(a, b), self.zip_high_i32x4(a, b))
}
#[inline(always)]
fn deinterleave_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i32x4<Self>, i32x4<Self>) {
(self.unzip_low_i32x4(a, b), self.unzip_high_i32x4(a, b))
}
#[inline(always)]
fn select_i32x4(self, a: mask32x4<Self>, b: i32x4<Self>, c: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask32x4<Avx2>,
b: i32x4<Avx2>,
c: i32x4<Avx2>,
) -> i32x4<Avx2> {
_mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i32x8<Avx2> {
_mm256_setr_m128i(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn neg_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>) -> i32x4<Avx2> {
_mm_sub_epi32(_mm_setzero_si128(), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4<Self>; 4usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, src: &[i32; 16usize]) -> [i32x4<Avx2>; 4usize] {
let (chunks, []) = src.as_chunks::<4usize>() else {
unreachable!()
};
let v0: __m128i =
crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[0]);
let v1: __m128i =
crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[1]);
let v2: __m128i =
crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[2]);
let v3: __m128i =
crate::transmute::checked_transmute_copy::<[i32; 4usize], __m128i>(&chunks[3]);
let tmp0 = _mm_unpacklo_epi32(v0, v1);
let tmp1 = _mm_unpackhi_epi32(v0, v1);
let tmp2 = _mm_unpacklo_epi32(v2, v3);
let tmp3 = _mm_unpackhi_epi32(v2, v3);
let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
[
out0.simd_into(token),
out1.simd_into(token),
out2.simd_into(token),
out3.simd_into(token),
]
}
);
kernel(self, src)
}
#[inline(always)]
fn store_four_interleaved_i32x4(
self,
vectors: [i32x4<Self>; 4usize],
dest: &mut [i32; 16usize],
) -> () {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
vectors: [i32x4<Avx2>; 4usize],
dest: &mut [i32; 16usize],
) -> () {
let _ = token;
let v0: __m128i = vectors[0].into();
let v1: __m128i = vectors[1].into();
let v2: __m128i = vectors[2].into();
let v3: __m128i = vectors[3].into();
let tmp0 = _mm_unpacklo_epi32(v0, v1);
let tmp1 = _mm_unpackhi_epi32(v0, v1);
let tmp2 = _mm_unpacklo_epi32(v2, v3);
let tmp3 = _mm_unpackhi_epi32(v2, v3);
let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
let (chunks, []) = dest.as_chunks_mut::<4usize>() else {
unreachable!()
};
crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>(
out0,
&mut chunks[0],
);
crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>(
out1,
&mut chunks[1],
);
crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>(
out2,
&mut chunks[2],
);
crate::transmute::checked_transmute_store::<__m128i, [i32; 4usize]>(
out3,
&mut chunks[3],
);
}
);
kernel(self, vectors, dest);
}
#[inline(always)]
fn widen_i32x4(self, a: i32x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>) -> (i64x2<Avx2>, i64x2<Avx2>) {
let raw = a.into();
(
_mm_cvtepi32_epi64(raw).simd_into(token),
_mm_cvtepi32_epi64(_mm_srli_si128::<8>(raw)).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i16x8<Avx2> {
let mask = _mm_set1_epi32(0xffff);
_mm_packus_epi32(_mm_and_si128(a.into(), mask), _mm_and_si128(b.into(), mask))
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>, b: i32x4<Avx2>) -> i16x8<Avx2> {
_mm_packs_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn relaxed_narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= i16::MIN as i32 && value <= i16::MAX as i32 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.saturating_narrow_i32x4(a, b)
}
#[inline(always)]
fn cvt_f32_i32x4(self, a: i32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x4<Avx2>) -> f32x4<Avx2> {
_mm_cvtepi32_ps(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_u32x4(self, val: u32) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: u32) -> u32x4<Avx2> {
_mm_set1_epi32(val.cast_signed()).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_u32x4<const SHIFT: usize>(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = dyn_alignr_128(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
_mm_add_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
_mm_sub_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
_mm_mullo_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
_mm_and_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
_mm_or_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
_mm_xor_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
a ^ !0
}
#[inline(always)]
fn shl_u32x4(self, a: u32x4<Self>, shift: u32) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, shift: u32) -> u32x4<Avx2> {
_mm_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
_mm_sllv_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_u32x4(self, a: u32x4<Self>, shift: u32) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, shift: u32) -> u32x4<Avx2> {
_mm_srl_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
_mm_srlv_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
_mm_max_epu32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
_mm_min_epu32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> mask32x4<Avx2> {
_mm_cmpeq_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> mask32x4<Avx2> {
{
let sign_bit = _mm_set1_epi32(0x80000000u32.cast_signed());
let lhs_signed = _mm_xor_si128(b.into(), sign_bit);
let rhs_signed = _mm_xor_si128(a.into(), sign_bit);
_mm_cmpgt_epi32(lhs_signed, rhs_signed)
}
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> mask32x4<Avx2> {
_mm_cmpeq_epi32(_mm_min_epu32(a.into(), b.into()), a.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
_mm_unpacklo_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
_mm_unpackhi_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
let t1 = _mm_shuffle_epi32::<0b11_01_10_00>(a.into());
let t2 = _mm_shuffle_epi32::<0b11_01_10_00>(b.into());
_mm_unpacklo_epi64(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x4<Avx2> {
let t1 = _mm_shuffle_epi32::<0b11_01_10_00>(a.into());
let t2 = _mm_shuffle_epi32::<0b11_01_10_00>(b.into());
_mm_unpackhi_epi64(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u32x4<Self>, u32x4<Self>) {
(self.zip_low_u32x4(a, b), self.zip_high_u32x4(a, b))
}
#[inline(always)]
fn deinterleave_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u32x4<Self>, u32x4<Self>) {
(self.unzip_low_u32x4(a, b), self.unzip_high_u32x4(a, b))
}
#[inline(always)]
fn select_u32x4(self, a: mask32x4<Self>, b: u32x4<Self>, c: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask32x4<Avx2>,
b: u32x4<Avx2>,
c: u32x4<Avx2>,
) -> u32x4<Avx2> {
_mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u32x8<Avx2> {
_mm256_setr_m128i(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4<Self>; 4usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, src: &[u32; 16usize]) -> [u32x4<Avx2>; 4usize] {
let (chunks, []) = src.as_chunks::<4usize>() else {
unreachable!()
};
let v0: __m128i =
crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[0]);
let v1: __m128i =
crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[1]);
let v2: __m128i =
crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[2]);
let v3: __m128i =
crate::transmute::checked_transmute_copy::<[u32; 4usize], __m128i>(&chunks[3]);
let tmp0 = _mm_unpacklo_epi32(v0, v1);
let tmp1 = _mm_unpackhi_epi32(v0, v1);
let tmp2 = _mm_unpacklo_epi32(v2, v3);
let tmp3 = _mm_unpackhi_epi32(v2, v3);
let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
[
out0.simd_into(token),
out1.simd_into(token),
out2.simd_into(token),
out3.simd_into(token),
]
}
);
kernel(self, src)
}
#[inline(always)]
fn store_four_interleaved_u32x4(
self,
vectors: [u32x4<Self>; 4usize],
dest: &mut [u32; 16usize],
) -> () {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
vectors: [u32x4<Avx2>; 4usize],
dest: &mut [u32; 16usize],
) -> () {
let _ = token;
let v0: __m128i = vectors[0].into();
let v1: __m128i = vectors[1].into();
let v2: __m128i = vectors[2].into();
let v3: __m128i = vectors[3].into();
let tmp0 = _mm_unpacklo_epi32(v0, v1);
let tmp1 = _mm_unpackhi_epi32(v0, v1);
let tmp2 = _mm_unpacklo_epi32(v2, v3);
let tmp3 = _mm_unpackhi_epi32(v2, v3);
let out0 = _mm_unpacklo_epi64(tmp0, tmp2);
let out1 = _mm_unpackhi_epi64(tmp0, tmp2);
let out2 = _mm_unpacklo_epi64(tmp1, tmp3);
let out3 = _mm_unpackhi_epi64(tmp1, tmp3);
let (chunks, []) = dest.as_chunks_mut::<4usize>() else {
unreachable!()
};
crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>(
out0,
&mut chunks[0],
);
crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>(
out1,
&mut chunks[1],
);
crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>(
out2,
&mut chunks[2],
);
crate::transmute::checked_transmute_store::<__m128i, [u32; 4usize]>(
out3,
&mut chunks[3],
);
}
);
kernel(self, vectors, dest);
}
#[inline(always)]
fn widen_u32x4(self, a: u32x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>) -> (u64x2<Avx2>, u64x2<Avx2>) {
let raw = a.into();
(
_mm_cvtepu32_epi64(raw).simd_into(token),
_mm_cvtepu32_epi64(_mm_srli_si128::<8>(raw)).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u16x8<Avx2> {
let mask = _mm_set1_epi32(0xffff);
_mm_packus_epi32(_mm_and_si128(a.into(), mask), _mm_and_si128(b.into(), mask))
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>, b: u32x4<Avx2>) -> u16x8<Avx2> {
let max = _mm_set1_epi32(u16::MAX as i32);
_mm_packus_epi32(_mm_min_epu32(a.into(), max), _mm_min_epu32(b.into(), max))
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn relaxed_narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= u16::MIN as u32 && value <= u16::MAX as u32 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.narrow_u32x4(a, b)
}
#[inline(always)]
fn cvt_f32_u32x4(self, a: u32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x4<Avx2>) -> f32x4<Avx2> {
let a = a.into();
let lo = _mm_blend_epi16::<0xAA>(a, _mm_set1_epi32(0x4B000000));
let hi =
_mm_blend_epi16::<0xAA>(_mm_srli_epi32::<16>(a), _mm_set1_epi32(0x53000000));
let fhi = _mm_sub_ps(
_mm_castsi128_ps(hi),
_mm_set1_ps(f32::from_bits(0x53000080)),
);
let result = _mm_add_ps(_mm_castsi128_ps(lo), fhi);
result.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_mask32x4(self, val: bool) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: bool) -> mask32x4<Avx2> {
let val: i32 = if val { !0 } else { 0 };
_mm_set1_epi32(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn from_bitmask_mask32x4(self, bits: u64) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, bits: u64) -> mask32x4<Avx2> {
{
let bit_lanes = _mm_set1_epi32(bits as i32);
let bit_mask = _mm_setr_epi32(1, 2, 4, 8);
_mm_cmpeq_epi32(_mm_and_si128(bit_lanes, bit_mask), bit_mask)
}
.simd_into(token)
}
);
kernel(self, bits)
}
#[inline(always)]
fn to_bitmask_mask32x4(self, a: mask32x4<Self>) -> u64 {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask32x4<Avx2>) -> u64 {
_mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 as u64
}
);
kernel(self, a)
}
#[inline(always)]
fn set_mask32x4(self, a: &mut mask32x4<Self>, index: usize, value: bool) -> () {
assert!(
index < 4usize,
"mask lane index {index} is out of bounds for {} lanes",
4usize
);
let mut lanes: [i32; 4usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn and_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask32x4<Avx2>, b: mask32x4<Avx2>) -> mask32x4<Avx2> {
_mm_and_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask32x4<Avx2>, b: mask32x4<Avx2>) -> mask32x4<Avx2> {
_mm_or_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask32x4<Avx2>, b: mask32x4<Avx2>) -> mask32x4<Avx2> {
_mm_xor_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_mask32x4(self, a: mask32x4<Self>) -> mask32x4<Self> {
self.xor_mask32x4(a, self.splat_mask32x4(true))
}
#[inline(always)]
fn select_mask32x4(
self,
a: mask32x4<Self>,
b: mask32x4<Self>,
c: mask32x4<Self>,
) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask32x4<Avx2>,
b: mask32x4<Avx2>,
c: mask32x4<Avx2>,
) -> mask32x4<Avx2> {
_mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn simd_eq_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask32x4<Avx2>, b: mask32x4<Avx2>) -> mask32x4<Avx2> {
_mm_cmpeq_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn any_true_mask32x4(self, a: mask32x4<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask32x4<Avx2>) -> bool {
_mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 != 0
}
);
kernel(self, a)
}
#[inline(always)]
fn all_true_mask32x4(self, a: mask32x4<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask32x4<Avx2>) -> bool {
_mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 == 0b1111
}
);
kernel(self, a)
}
#[inline(always)]
fn any_false_mask32x4(self, a: mask32x4<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask32x4<Avx2>) -> bool {
_mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 != 0b1111
}
);
kernel(self, a)
}
#[inline(always)]
fn all_false_mask32x4(self, a: mask32x4<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask32x4<Avx2>) -> bool {
_mm_movemask_ps(_mm_castsi128_ps(a.into())) as u32 == 0
}
);
kernel(self, a)
}
#[inline(always)]
fn combine_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask32x4<Avx2>, b: mask32x4<Avx2>) -> mask32x8<Avx2> {
_mm256_setr_m128i(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn splat_f64x2(self, val: f64) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: f64) -> f64x2<Avx2> {
_mm_set1_pd(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_f64x2<const SHIFT: usize>(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
if SHIFT >= 2usize {
return b;
}
let result = dyn_alignr_128(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn abs_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>) -> f64x2<Avx2> {
_mm_andnot_pd(_mm_set1_pd(-0.0), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>) -> f64x2<Avx2> {
_mm_xor_pd(a.into(), _mm_set1_pd(-0.0)).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn sqrt_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>) -> f64x2<Avx2> {
_mm_sqrt_pd(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn approximate_recip_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
1.0 / a
}
#[inline(always)]
fn add_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
_mm_add_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
_mm_sub_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
_mm_mul_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn div_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
_mm_div_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn copysign_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
let mask = _mm_set1_pd(-0.0);
_mm_or_pd(_mm_and_pd(mask, b.into()), _mm_andnot_pd(mask, a.into()))
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
_mm_max_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
_mm_min_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
let intermediate = _mm_max_pd(a.into(), b.into());
let b_is_nan = _mm_cmpunord_pd(b.into(), b.into());
_mm_blendv_pd(intermediate, a.into(), b_is_nan).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
let intermediate = _mm_min_pd(a.into(), b.into());
let b_is_nan = _mm_cmpunord_pd(b.into(), b.into());
_mm_blendv_pd(intermediate, a.into(), b_is_nan).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> mask64x2<Avx2> {
_mm_castpd_si128(_mm_cmpeq_pd(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> mask64x2<Avx2> {
_mm_castpd_si128(_mm_cmplt_pd(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> mask64x2<Avx2> {
_mm_castpd_si128(_mm_cmple_pd(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
_mm_unpacklo_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
_mm_unpackhi_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
_mm_shuffle_pd::<0b00>(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x2<Avx2> {
_mm_shuffle_pd::<0b11>(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> (f64x2<Self>, f64x2<Self>) {
(self.zip_low_f64x2(a, b), self.zip_high_f64x2(a, b))
}
#[inline(always)]
fn deinterleave_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> (f64x2<Self>, f64x2<Self>) {
(self.unzip_low_f64x2(a, b), self.unzip_high_f64x2(a, b))
}
#[inline(always)]
fn mul_add_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>, c: f64x2<Avx2>) -> f64x2<Avx2> {
_mm_fmadd_pd(a.into(), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn mul_sub_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>, c: f64x2<Avx2>) -> f64x2<Avx2> {
_mm_fmsub_pd(a.into(), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn floor_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>) -> f64x2<Avx2> {
_mm_round_pd::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn ceil_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>) -> f64x2<Avx2> {
_mm_round_pd::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn round_ties_even_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>) -> f64x2<Avx2> {
_mm_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn fract_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
a - self.trunc_f64x2(a)
}
#[inline(always)]
fn trunc_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>) -> f64x2<Avx2> {
_mm_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn select_f64x2(self, a: mask64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask64x2<Avx2>,
b: f64x2<Avx2>,
c: f64x2<Avx2>,
) -> f64x2<Avx2> {
_mm_blendv_pd(c.into(), b.into(), _mm_castsi128_pd(a.into())).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f64x4<Avx2> {
_mm256_setr_m128d(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2<Self>; 4usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, src: &[f64; 8usize]) -> [f64x2<Avx2>; 4usize] {
let (chunks, []) = src.as_chunks::<4>() else {
unreachable!()
};
let v0: __m256d =
crate::transmute::checked_transmute_copy::<[f64; 4], __m256d>(&chunks[0]);
let v1: __m256d =
crate::transmute::checked_transmute_copy::<[f64; 4], __m256d>(&chunks[1]);
let lo = _mm256_unpacklo_pd(v0, v1);
let hi = _mm256_unpackhi_pd(v0, v1);
let out0 = _mm256_permute2f128_pd::<0x20>(lo, hi);
let out1 = _mm256_permute2f128_pd::<0x31>(lo, hi);
let outputs: [__m128d; 4] = crate::transmute::checked_transmute_copy(&[out0, out1]);
[
outputs[0].simd_into(token),
outputs[1].simd_into(token),
outputs[2].simd_into(token),
outputs[3].simd_into(token),
]
}
);
kernel(self, src)
}
#[inline(always)]
fn store_four_interleaved_f64x2(
self,
vectors: [f64x2<Self>; 4usize],
dest: &mut [f64; 8usize],
) -> () {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, vectors: [f64x2<Avx2>; 4usize], dest: &mut [f64; 8usize]) -> () {
let _ = token;
let inputs: [__m128d; 4] = [
vectors[0].into(),
vectors[1].into(),
vectors[2].into(),
vectors[3].into(),
];
let wide_inputs: [__m256d; 2] = crate::transmute::checked_transmute_copy(&inputs);
let v0 = wide_inputs[0];
let v1 = wide_inputs[1];
let lo = _mm256_permute2f128_pd::<0x20>(v0, v1);
let hi = _mm256_permute2f128_pd::<0x31>(v0, v1);
let out0 = _mm256_unpacklo_pd(lo, hi);
let out1 = _mm256_unpackhi_pd(lo, hi);
let (chunks, []) = dest.as_chunks_mut::<4>() else {
unreachable!()
};
crate::transmute::checked_transmute_store::<__m256d, [f64; 4]>(
out0,
&mut chunks[0],
);
crate::transmute::checked_transmute_store::<__m256d, [f64; 4]>(
out1,
&mut chunks[1],
);
}
);
kernel(self, vectors, dest);
}
#[inline(always)]
fn narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x2<Avx2>, b: f64x2<Avx2>) -> f32x4<Avx2> {
let low = _mm_cvtpd_ps(a.into());
let high = _mm_cvtpd_ps(b.into());
_mm_movelh_ps(low, high).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
self.narrow_f64x2(a, b)
}
#[inline(always)]
fn relaxed_narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
self.narrow_f64x2(a, b)
}
#[inline(always)]
fn cvt_u64_f64x2(self, a: f64x2<Self>) -> u64x2<Self> {
[a[0usize] as u64, a[1usize] as u64].simd_into(self)
}
#[inline(always)]
fn cvt_u64_precise_f64x2(self, a: f64x2<Self>) -> u64x2<Self> {
[a[0usize] as u64, a[1usize] as u64].simd_into(self)
}
#[inline(always)]
fn cvt_i64_f64x2(self, a: f64x2<Self>) -> i64x2<Self> {
[a[0usize] as i64, a[1usize] as i64].simd_into(self)
}
#[inline(always)]
fn cvt_i64_precise_f64x2(self, a: f64x2<Self>) -> i64x2<Self> {
[a[0usize] as i64, a[1usize] as i64].simd_into(self)
}
#[inline(always)]
fn splat_i64x2(self, val: i64) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: i64) -> i64x2<Avx2> {
_mm_set1_epi64x(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_i64x2<const SHIFT: usize>(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
if SHIFT >= 2usize {
return b;
}
let result = dyn_alignr_128(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
_mm_add_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
_mm_sub_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
[
i64::wrapping_mul(a[0usize], b[0usize]),
i64::wrapping_mul(a[1usize], b[1usize]),
]
.simd_into(self)
}
#[inline(always)]
fn and_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
_mm_and_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
_mm_or_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
_mm_xor_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
a ^ !0
}
#[inline(always)]
fn shl_i64x2(self, a: i64x2<Self>, shift: u32) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x2<Avx2>, shift: u32) -> i64x2<Avx2> {
_mm_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
_mm_sllv_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_i64x2(self, a: i64x2<Self>, shift: u32) -> i64x2<Self> {
[
i64::wrapping_shr(a[0usize], shift),
i64::wrapping_shr(a[1usize], shift),
]
.simd_into(self)
}
#[inline(always)]
fn shrv_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
let value = a.into();
let counts = b.into();
let bias = _mm_set1_epi64x(i64::MIN);
let shifted_bias = _mm_srlv_epi64(bias, counts);
let shifted = _mm_srlv_epi64(value, counts);
_mm_sub_epi64(_mm_xor_si128(shifted, shifted_bias), shifted_bias).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
[
i64::max(a[0usize], b[0usize]),
i64::max(a[1usize], b[1usize]),
]
.simd_into(self)
}
#[inline(always)]
fn min_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
[
i64::min(a[0usize], b[0usize]),
i64::min(a[1usize], b[1usize]),
]
.simd_into(self)
}
#[inline(always)]
fn simd_eq_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> mask64x2<Avx2> {
_mm_cmpeq_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
[
-(i64::lt(&a[0usize], &b[0usize]) as i64),
-(i64::lt(&a[1usize], &b[1usize]) as i64),
]
.simd_into(self)
}
#[inline(always)]
fn simd_le_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
[
-(i64::le(&a[0usize], &b[0usize]) as i64),
-(i64::le(&a[1usize], &b[1usize]) as i64),
]
.simd_into(self)
}
#[inline(always)]
fn zip_low_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
_mm_unpacklo_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
_mm_unpackhi_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
_mm_unpacklo_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x2<Avx2> {
_mm_unpackhi_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> (i64x2<Self>, i64x2<Self>) {
(self.zip_low_i64x2(a, b), self.zip_high_i64x2(a, b))
}
#[inline(always)]
fn deinterleave_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> (i64x2<Self>, i64x2<Self>) {
(self.unzip_low_i64x2(a, b), self.unzip_high_i64x2(a, b))
}
#[inline(always)]
fn select_i64x2(self, a: mask64x2<Self>, b: i64x2<Self>, c: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask64x2<Avx2>,
b: i64x2<Avx2>,
c: i64x2<Avx2>,
) -> i64x2<Avx2> {
_mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i64x4<Avx2> {
_mm256_setr_m128i(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn neg_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x2<Avx2>) -> i64x2<Avx2> {
_mm_sub_epi64(_mm_setzero_si128(), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2<Self>; 4usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, src: &[i64; 8usize]) -> [i64x2<Avx2>; 4usize] {
let (chunks, []) = src.as_chunks::<4>() else {
unreachable!()
};
let v0: __m256i =
crate::transmute::checked_transmute_copy::<[i64; 4], __m256i>(&chunks[0]);
let v1: __m256i =
crate::transmute::checked_transmute_copy::<[i64; 4], __m256i>(&chunks[1]);
let lo = _mm256_unpacklo_epi64(v0, v1);
let hi = _mm256_unpackhi_epi64(v0, v1);
let out0 = _mm256_permute2x128_si256::<0x20>(lo, hi);
let out1 = _mm256_permute2x128_si256::<0x31>(lo, hi);
let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&[out0, out1]);
[
outputs[0].simd_into(token),
outputs[1].simd_into(token),
outputs[2].simd_into(token),
outputs[3].simd_into(token),
]
}
);
kernel(self, src)
}
#[inline(always)]
fn store_four_interleaved_i64x2(
self,
vectors: [i64x2<Self>; 4usize],
dest: &mut [i64; 8usize],
) -> () {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, vectors: [i64x2<Avx2>; 4usize], dest: &mut [i64; 8usize]) -> () {
let _ = token;
let inputs: [__m128i; 4] = [
vectors[0].into(),
vectors[1].into(),
vectors[2].into(),
vectors[3].into(),
];
let wide_inputs: [__m256i; 2] = crate::transmute::checked_transmute_copy(&inputs);
let v0 = wide_inputs[0];
let v1 = wide_inputs[1];
let lo = _mm256_permute2x128_si256::<0x20>(v0, v1);
let hi = _mm256_permute2x128_si256::<0x31>(v0, v1);
let out0 = _mm256_unpacklo_epi64(lo, hi);
let out1 = _mm256_unpackhi_epi64(lo, hi);
let (chunks, []) = dest.as_chunks_mut::<4>() else {
unreachable!()
};
crate::transmute::checked_transmute_store::<__m256i, [i64; 4]>(
out0,
&mut chunks[0],
);
crate::transmute::checked_transmute_store::<__m256i, [i64; 4]>(
out1,
&mut chunks[1],
);
}
);
kernel(self, vectors, dest);
}
#[inline(always)]
fn narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i32x4<Avx2> {
_mm_castps_si128(_mm_shuffle_ps::<0x88>(
_mm_castsi128_ps(a.into()),
_mm_castsi128_ps(b.into()),
))
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x2<Avx2>, b: i64x2<Avx2>) -> i32x4<Avx2> {
let a = a.into();
let b = b.into();
let low = _mm_castps_si128(_mm_shuffle_ps::<0x88>(
_mm_castsi128_ps(a),
_mm_castsi128_ps(b),
));
let high = _mm_castps_si128(_mm_shuffle_ps::<0xdd>(
_mm_castsi128_ps(a),
_mm_castsi128_ps(b),
));
let low_sign = _mm_srai_epi32::<31>(low);
let fits = _mm_cmpeq_epi32(high, low_sign);
let high_sign = _mm_srai_epi32::<31>(high);
let bound = _mm_xor_si128(high_sign, _mm_set1_epi32(i32::MAX));
_mm_blendv_epi8(bound, low, fits).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn relaxed_narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= i32::MIN as i64 && value <= i32::MAX as i64 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.narrow_i64x2(a, b)
}
#[inline(always)]
fn cvt_f64_i64x2(self, a: i64x2<Self>) -> f64x2<Self> {
[a[0usize] as f64, a[1usize] as f64].simd_into(self)
}
#[inline(always)]
fn splat_u64x2(self, val: u64) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: u64) -> u64x2<Avx2> {
_mm_set1_epi64x(val.cast_signed()).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_u64x2<const SHIFT: usize>(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
if SHIFT >= 2usize {
return b;
}
let result = dyn_alignr_128(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
_mm_add_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
_mm_sub_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
[
u64::wrapping_mul(a[0usize], b[0usize]),
u64::wrapping_mul(a[1usize], b[1usize]),
]
.simd_into(self)
}
#[inline(always)]
fn and_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
_mm_and_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
_mm_or_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
_mm_xor_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
a ^ !0
}
#[inline(always)]
fn shl_u64x2(self, a: u64x2<Self>, shift: u32) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x2<Avx2>, shift: u32) -> u64x2<Avx2> {
_mm_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
_mm_sllv_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_u64x2(self, a: u64x2<Self>, shift: u32) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x2<Avx2>, shift: u32) -> u64x2<Avx2> {
_mm_srl_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
_mm_srlv_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
[
u64::max(a[0usize], b[0usize]),
u64::max(a[1usize], b[1usize]),
]
.simd_into(self)
}
#[inline(always)]
fn min_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
[
u64::min(a[0usize], b[0usize]),
u64::min(a[1usize], b[1usize]),
]
.simd_into(self)
}
#[inline(always)]
fn simd_eq_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> mask64x2<Avx2> {
_mm_cmpeq_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
[
-(u64::lt(&a[0usize], &b[0usize]) as i64),
-(u64::lt(&a[1usize], &b[1usize]) as i64),
]
.simd_into(self)
}
#[inline(always)]
fn simd_le_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
[
-(u64::le(&a[0usize], &b[0usize]) as i64),
-(u64::le(&a[1usize], &b[1usize]) as i64),
]
.simd_into(self)
}
#[inline(always)]
fn zip_low_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
_mm_unpacklo_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
_mm_unpackhi_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
_mm_unpacklo_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x2<Avx2> {
_mm_unpackhi_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> (u64x2<Self>, u64x2<Self>) {
(self.zip_low_u64x2(a, b), self.zip_high_u64x2(a, b))
}
#[inline(always)]
fn deinterleave_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> (u64x2<Self>, u64x2<Self>) {
(self.unzip_low_u64x2(a, b), self.unzip_high_u64x2(a, b))
}
#[inline(always)]
fn select_u64x2(self, a: mask64x2<Self>, b: u64x2<Self>, c: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask64x2<Avx2>,
b: u64x2<Avx2>,
c: u64x2<Avx2>,
) -> u64x2<Avx2> {
_mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u64x4<Avx2> {
_mm256_setr_m128i(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2<Self>; 4usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, src: &[u64; 8usize]) -> [u64x2<Avx2>; 4usize] {
let (chunks, []) = src.as_chunks::<4>() else {
unreachable!()
};
let v0: __m256i =
crate::transmute::checked_transmute_copy::<[u64; 4], __m256i>(&chunks[0]);
let v1: __m256i =
crate::transmute::checked_transmute_copy::<[u64; 4], __m256i>(&chunks[1]);
let lo = _mm256_unpacklo_epi64(v0, v1);
let hi = _mm256_unpackhi_epi64(v0, v1);
let out0 = _mm256_permute2x128_si256::<0x20>(lo, hi);
let out1 = _mm256_permute2x128_si256::<0x31>(lo, hi);
let outputs: [__m128i; 4] = crate::transmute::checked_transmute_copy(&[out0, out1]);
[
outputs[0].simd_into(token),
outputs[1].simd_into(token),
outputs[2].simd_into(token),
outputs[3].simd_into(token),
]
}
);
kernel(self, src)
}
#[inline(always)]
fn store_four_interleaved_u64x2(
self,
vectors: [u64x2<Self>; 4usize],
dest: &mut [u64; 8usize],
) -> () {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, vectors: [u64x2<Avx2>; 4usize], dest: &mut [u64; 8usize]) -> () {
let _ = token;
let inputs: [__m128i; 4] = [
vectors[0].into(),
vectors[1].into(),
vectors[2].into(),
vectors[3].into(),
];
let wide_inputs: [__m256i; 2] = crate::transmute::checked_transmute_copy(&inputs);
let v0 = wide_inputs[0];
let v1 = wide_inputs[1];
let lo = _mm256_permute2x128_si256::<0x20>(v0, v1);
let hi = _mm256_permute2x128_si256::<0x31>(v0, v1);
let out0 = _mm256_unpacklo_epi64(lo, hi);
let out1 = _mm256_unpackhi_epi64(lo, hi);
let (chunks, []) = dest.as_chunks_mut::<4>() else {
unreachable!()
};
crate::transmute::checked_transmute_store::<__m256i, [u64; 4]>(
out0,
&mut chunks[0],
);
crate::transmute::checked_transmute_store::<__m256i, [u64; 4]>(
out1,
&mut chunks[1],
);
}
);
kernel(self, vectors, dest);
}
#[inline(always)]
fn narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u32x4<Avx2> {
_mm_castps_si128(_mm_shuffle_ps::<0x88>(
_mm_castsi128_ps(a.into()),
_mm_castsi128_ps(b.into()),
))
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x2<Avx2>, b: u64x2<Avx2>) -> u32x4<Avx2> {
let a = a.into();
let b = b.into();
let low = _mm_castps_si128(_mm_shuffle_ps::<0x88>(
_mm_castsi128_ps(a),
_mm_castsi128_ps(b),
));
let high = _mm_castps_si128(_mm_shuffle_ps::<0xdd>(
_mm_castsi128_ps(a),
_mm_castsi128_ps(b),
));
let zero = _mm_setzero_si128();
let fits = _mm_cmpeq_epi32(high, zero);
let ones = _mm_cmpeq_epi32(zero, zero);
_mm_or_si128(low, _mm_xor_si128(fits, ones)).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn relaxed_narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= u32::MIN as u64 && value <= u32::MAX as u64 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.narrow_u64x2(a, b)
}
#[inline(always)]
fn cvt_f64_u64x2(self, a: u64x2<Self>) -> f64x2<Self> {
[a[0usize] as f64, a[1usize] as f64].simd_into(self)
}
#[inline(always)]
fn splat_mask64x2(self, val: bool) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: bool) -> mask64x2<Avx2> {
let val: i64 = if val { !0 } else { 0 };
_mm_set1_epi64x(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, bits: u64) -> mask64x2<Avx2> {
{
let bit_lanes = _mm_set1_epi64x(bits.cast_signed());
let bit_mask = _mm_set_epi64x(2, 1);
_mm_cmpeq_epi64(_mm_and_si128(bit_lanes, bit_mask), bit_mask)
}
.simd_into(token)
}
);
kernel(self, bits)
}
#[inline(always)]
fn to_bitmask_mask64x2(self, a: mask64x2<Self>) -> u64 {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask64x2<Avx2>) -> u64 {
_mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 as u64
}
);
kernel(self, a)
}
#[inline(always)]
fn set_mask64x2(self, a: &mut mask64x2<Self>, index: usize, value: bool) -> () {
assert!(
index < 2usize,
"mask lane index {index} is out of bounds for {} lanes",
2usize
);
let mut lanes: [i64; 2usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn and_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask64x2<Avx2>, b: mask64x2<Avx2>) -> mask64x2<Avx2> {
_mm_and_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask64x2<Avx2>, b: mask64x2<Avx2>) -> mask64x2<Avx2> {
_mm_or_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask64x2<Avx2>, b: mask64x2<Avx2>) -> mask64x2<Avx2> {
_mm_xor_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_mask64x2(self, a: mask64x2<Self>) -> mask64x2<Self> {
self.xor_mask64x2(a, self.splat_mask64x2(true))
}
#[inline(always)]
fn select_mask64x2(
self,
a: mask64x2<Self>,
b: mask64x2<Self>,
c: mask64x2<Self>,
) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask64x2<Avx2>,
b: mask64x2<Avx2>,
c: mask64x2<Avx2>,
) -> mask64x2<Avx2> {
_mm_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn simd_eq_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask64x2<Avx2>, b: mask64x2<Avx2>) -> mask64x2<Avx2> {
_mm_cmpeq_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn any_true_mask64x2(self, a: mask64x2<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask64x2<Avx2>) -> bool {
_mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 != 0
}
);
kernel(self, a)
}
#[inline(always)]
fn all_true_mask64x2(self, a: mask64x2<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask64x2<Avx2>) -> bool {
_mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 == 0b11
}
);
kernel(self, a)
}
#[inline(always)]
fn any_false_mask64x2(self, a: mask64x2<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask64x2<Avx2>) -> bool {
_mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 != 0b11
}
);
kernel(self, a)
}
#[inline(always)]
fn all_false_mask64x2(self, a: mask64x2<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask64x2<Avx2>) -> bool {
_mm_movemask_pd(_mm_castsi128_pd(a.into())) as u32 == 0
}
);
kernel(self, a)
}
#[inline(always)]
fn combine_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask64x2<Avx2>, b: mask64x2<Avx2>) -> mask64x4<Avx2> {
_mm256_setr_m128i(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn splat_f32x8(self, val: f32) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: f32) -> f32x8<Avx2> {
_mm256_set1_ps(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_f32x8<const SHIFT: usize>(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = cross_block_alignr_256x1(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn slide_within_blocks_f32x8<const SHIFT: usize>(
self,
a: f32x8<Self>,
b: f32x8<Self>,
) -> f32x8<Self> {
if SHIFT >= 4usize {
return b;
}
let result = dyn_alignr_256(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn abs_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>) -> f32x8<Avx2> {
_mm256_andnot_ps(_mm256_set1_ps(-0.0), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>) -> f32x8<Avx2> {
_mm256_xor_ps(a.into(), _mm256_set1_ps(-0.0)).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn sqrt_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>) -> f32x8<Avx2> {
_mm256_sqrt_ps(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn approximate_recip_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>) -> f32x8<Avx2> {
_mm256_rcp_ps(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn add_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
_mm256_add_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
_mm256_sub_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
_mm256_mul_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn div_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
_mm256_div_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn copysign_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
let mask = _mm256_set1_ps(-0.0);
_mm256_or_ps(
_mm256_and_ps(mask, b.into()),
_mm256_andnot_ps(mask, a.into()),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
_mm256_max_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
_mm256_min_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_precise_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
let intermediate = _mm256_max_ps(a.into(), b.into());
let b_is_nan = _mm256_cmp_ps::<3i32>(b.into(), b.into());
_mm256_blendv_ps(intermediate, a.into(), b_is_nan).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_precise_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
let intermediate = _mm256_min_ps(a.into(), b.into());
let b_is_nan = _mm256_cmp_ps::<3i32>(b.into(), b.into());
_mm256_blendv_ps(intermediate, a.into(), b_is_nan).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> mask32x8<Avx2> {
_mm256_castps_si256(_mm256_cmp_ps::<0i32>(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> mask32x8<Avx2> {
_mm256_castps_si256(_mm256_cmp_ps::<17i32>(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> mask32x8<Avx2> {
_mm256_castps_si256(_mm256_cmp_ps::<18i32>(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
let lo = _mm256_unpacklo_ps(a.into(), b.into());
let hi = _mm256_unpackhi_ps(a.into(), b.into());
_mm256_permute2f128_ps::<0b0010_0000>(lo, hi).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
let lo = _mm256_unpacklo_ps(a.into(), b.into());
let hi = _mm256_unpackhi_ps(a.into(), b.into());
_mm256_permute2f128_ps::<0b0011_0001>(lo, hi).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
let t1 =
_mm256_permutevar8x32_ps(a.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7));
let t2 =
_mm256_permutevar8x32_ps(b.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7));
_mm256_permute2f128_ps::<0b0010_0000>(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> f32x8<Avx2> {
let t1 =
_mm256_permutevar8x32_ps(a.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7));
let t2 =
_mm256_permutevar8x32_ps(b.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7));
_mm256_permute2f128_ps::<0b0011_0001>(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> (f32x8<Self>, f32x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> (f32x8<Avx2>, f32x8<Avx2>) {
let lo = _mm256_unpacklo_ps(a.into(), b.into());
let hi = _mm256_unpackhi_ps(a.into(), b.into());
(
_mm256_permute2f128_ps::<0b0010_0000>(lo, hi).simd_into(token),
_mm256_permute2f128_ps::<0b0011_0001>(lo, hi).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> (f32x8<Self>, f32x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>) -> (f32x8<Avx2>, f32x8<Avx2>) {
let t1 =
_mm256_permutevar8x32_ps(a.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7));
let t2 =
_mm256_permutevar8x32_ps(b.into(), _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7));
(
_mm256_permute2f128_ps::<0b0010_0000>(t1, t2).simd_into(token),
_mm256_permute2f128_ps::<0b0011_0001>(t1, t2).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_add_f32x8(self, a: f32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>, c: f32x8<Avx2>) -> f32x8<Avx2> {
_mm256_fmadd_ps(a.into(), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn mul_sub_f32x8(self, a: f32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>, b: f32x8<Avx2>, c: f32x8<Avx2>) -> f32x8<Avx2> {
_mm256_fmsub_ps(a.into(), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn floor_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>) -> f32x8<Avx2> {
_mm256_round_ps::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn ceil_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>) -> f32x8<Avx2> {
_mm256_round_ps::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn round_ties_even_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>) -> f32x8<Avx2> {
_mm256_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn fract_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
a - self.trunc_f32x8(a)
}
#[inline(always)]
fn trunc_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>) -> f32x8<Avx2> {
_mm256_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn select_f32x8(self, a: mask32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask32x8<Avx2>,
b: f32x8<Avx2>,
c: f32x8<Avx2>,
) -> f32x8<Avx2> {
_mm256_blendv_ps(c.into(), b.into(), _mm256_castsi256_ps(a.into())).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x16<Self> {
f32x16 {
val: crate::support::Aligned512([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn split_f32x8(self, a: f32x8<Self>) -> (f32x4<Self>, f32x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>) -> (f32x4<Avx2>, f32x4<Avx2>) {
(
_mm256_extractf128_ps::<0>(a.into()).simd_into(token),
_mm256_extractf128_ps::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn widen_f32x8(self, a: f32x8<Self>) -> (f64x4<Self>, f64x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>) -> (f64x4<Avx2>, f64x4<Avx2>) {
let raw = a.into();
(
_mm256_cvtps_pd(_mm256_castps256_ps128(raw)).simd_into(token),
_mm256_cvtps_pd(_mm256_extractf128_ps::<1>(raw)).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_u32_f32x8(self, a: f32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>) -> u32x8<Avx2> {
let mut converted = _mm256_cvttps_epi32(a.into());
let in_range = _mm256_cmp_ps::<17i32>(a.into(), _mm256_set1_ps(2147483648.0));
let all_in_range = _mm256_movemask_ps(in_range) == 0b11111111;
if !all_in_range {
let excess = _mm256_sub_ps(a.into(), _mm256_set1_ps(2147483648.0));
let excess_converted = _mm256_cvttps_epi32(_mm256_andnot_ps(in_range, excess));
converted = _mm256_add_epi32(converted, excess_converted);
}
converted.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_u32_precise_f32x8(self, a: f32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>) -> u32x8<Avx2> {
let a = _mm256_max_ps(a.into(), _mm256_setzero_ps());
let mut converted = _mm256_cvttps_epi32(a);
let in_range = _mm256_cmp_ps::<17i32>(a, _mm256_set1_ps(2147483648.0));
let all_in_range = _mm256_movemask_ps(in_range) == 0b11111111;
if !all_in_range {
let exceeds_unsigned_range = _mm256_castps_si256(_mm256_cmp_ps::<17i32>(
_mm256_set1_ps(4294967040.0),
a,
));
let excess = _mm256_sub_ps(a, _mm256_set1_ps(2147483648.0));
let excess_converted = _mm256_cvttps_epi32(_mm256_andnot_ps(in_range, excess));
converted = _mm256_add_epi32(converted, excess_converted);
converted = _mm256_blendv_epi8(
converted,
_mm256_set1_epi32(u32::MAX.cast_signed()),
exceeds_unsigned_range,
);
}
converted.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_i32_f32x8(self, a: f32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>) -> i32x8<Avx2> {
_mm256_cvttps_epi32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_i32_precise_f32x8(self, a: f32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f32x8<Avx2>) -> i32x8<Avx2> {
let a = a.into();
let mut converted = _mm256_cvttps_epi32(a);
let in_range = _mm256_cmp_ps::<17i32>(a, _mm256_set1_ps(2147483648.0));
let all_in_range = _mm256_movemask_ps(in_range) == 0b11111111;
if !all_in_range {
converted = _mm256_blendv_epi8(
_mm256_set1_epi32(i32::MAX),
converted,
_mm256_castps_si256(in_range),
);
let is_not_nan = _mm256_castps_si256(_mm256_cmp_ps::<7i32>(a, a));
converted = _mm256_and_si256(converted, is_not_nan);
}
converted.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_i8x32(self, val: i8) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: i8) -> i8x32<Avx2> {
_mm256_set1_epi8(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_i8x32<const SHIFT: usize>(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
if SHIFT >= 32usize {
return b;
}
let result = cross_block_alignr_256x1(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn slide_within_blocks_i8x32<const SHIFT: usize>(
self,
a: i8x32<Self>,
b: i8x32<Self>,
) -> i8x32<Self> {
if SHIFT >= 16usize {
return b;
}
let result = dyn_alignr_256(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn add_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
_mm256_add_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
_mm256_sub_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
let dst_even = _mm256_mullo_epi16(a.into(), b.into());
let dst_odd = _mm256_mullo_epi16(
_mm256_srli_epi16::<8>(a.into()),
_mm256_srli_epi16::<8>(b.into()),
);
_mm256_or_si256(
_mm256_slli_epi16(dst_odd, 8),
_mm256_and_si256(dst_even, _mm256_set1_epi16(0xFF)),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
_mm256_and_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
_mm256_or_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
_mm256_xor_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i8x32(self, a: i8x32<Self>) -> i8x32<Self> {
a ^ !0
}
#[inline(always)]
fn shl_i8x32(self, a: i8x32<Self>, shift: u32) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>, shift: u32) -> i8x32<Avx2> {
let val = a.into();
let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
let byte_mask = _mm256_set1_epi8(mask_byte);
_mm256_sll_epi16(_mm256_and_si256(val, byte_mask), shift_count).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
[
i8::wrapping_shl(a[0usize], b[0usize] as u32),
i8::wrapping_shl(a[1usize], b[1usize] as u32),
i8::wrapping_shl(a[2usize], b[2usize] as u32),
i8::wrapping_shl(a[3usize], b[3usize] as u32),
i8::wrapping_shl(a[4usize], b[4usize] as u32),
i8::wrapping_shl(a[5usize], b[5usize] as u32),
i8::wrapping_shl(a[6usize], b[6usize] as u32),
i8::wrapping_shl(a[7usize], b[7usize] as u32),
i8::wrapping_shl(a[8usize], b[8usize] as u32),
i8::wrapping_shl(a[9usize], b[9usize] as u32),
i8::wrapping_shl(a[10usize], b[10usize] as u32),
i8::wrapping_shl(a[11usize], b[11usize] as u32),
i8::wrapping_shl(a[12usize], b[12usize] as u32),
i8::wrapping_shl(a[13usize], b[13usize] as u32),
i8::wrapping_shl(a[14usize], b[14usize] as u32),
i8::wrapping_shl(a[15usize], b[15usize] as u32),
i8::wrapping_shl(a[16usize], b[16usize] as u32),
i8::wrapping_shl(a[17usize], b[17usize] as u32),
i8::wrapping_shl(a[18usize], b[18usize] as u32),
i8::wrapping_shl(a[19usize], b[19usize] as u32),
i8::wrapping_shl(a[20usize], b[20usize] as u32),
i8::wrapping_shl(a[21usize], b[21usize] as u32),
i8::wrapping_shl(a[22usize], b[22usize] as u32),
i8::wrapping_shl(a[23usize], b[23usize] as u32),
i8::wrapping_shl(a[24usize], b[24usize] as u32),
i8::wrapping_shl(a[25usize], b[25usize] as u32),
i8::wrapping_shl(a[26usize], b[26usize] as u32),
i8::wrapping_shl(a[27usize], b[27usize] as u32),
i8::wrapping_shl(a[28usize], b[28usize] as u32),
i8::wrapping_shl(a[29usize], b[29usize] as u32),
i8::wrapping_shl(a[30usize], b[30usize] as u32),
i8::wrapping_shl(a[31usize], b[31usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn shr_i8x32(self, a: i8x32<Self>, shift: u32) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>, shift: u32) -> i8x32<Avx2> {
let val = a.into();
let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
let byte_mask = _mm256_set1_epi8(mask_byte);
let shifted = _mm256_srl_epi16(val, shift_count);
let result = {
let sign = _mm256_cmpgt_epi8(_mm256_setzero_si256(), val);
_mm256_or_si256(
_mm256_and_si256(shifted, byte_mask),
_mm256_andnot_si256(byte_mask, sign),
)
};
result.simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
[
i8::wrapping_shr(a[0usize], b[0usize] as u32),
i8::wrapping_shr(a[1usize], b[1usize] as u32),
i8::wrapping_shr(a[2usize], b[2usize] as u32),
i8::wrapping_shr(a[3usize], b[3usize] as u32),
i8::wrapping_shr(a[4usize], b[4usize] as u32),
i8::wrapping_shr(a[5usize], b[5usize] as u32),
i8::wrapping_shr(a[6usize], b[6usize] as u32),
i8::wrapping_shr(a[7usize], b[7usize] as u32),
i8::wrapping_shr(a[8usize], b[8usize] as u32),
i8::wrapping_shr(a[9usize], b[9usize] as u32),
i8::wrapping_shr(a[10usize], b[10usize] as u32),
i8::wrapping_shr(a[11usize], b[11usize] as u32),
i8::wrapping_shr(a[12usize], b[12usize] as u32),
i8::wrapping_shr(a[13usize], b[13usize] as u32),
i8::wrapping_shr(a[14usize], b[14usize] as u32),
i8::wrapping_shr(a[15usize], b[15usize] as u32),
i8::wrapping_shr(a[16usize], b[16usize] as u32),
i8::wrapping_shr(a[17usize], b[17usize] as u32),
i8::wrapping_shr(a[18usize], b[18usize] as u32),
i8::wrapping_shr(a[19usize], b[19usize] as u32),
i8::wrapping_shr(a[20usize], b[20usize] as u32),
i8::wrapping_shr(a[21usize], b[21usize] as u32),
i8::wrapping_shr(a[22usize], b[22usize] as u32),
i8::wrapping_shr(a[23usize], b[23usize] as u32),
i8::wrapping_shr(a[24usize], b[24usize] as u32),
i8::wrapping_shr(a[25usize], b[25usize] as u32),
i8::wrapping_shr(a[26usize], b[26usize] as u32),
i8::wrapping_shr(a[27usize], b[27usize] as u32),
i8::wrapping_shr(a[28usize], b[28usize] as u32),
i8::wrapping_shr(a[29usize], b[29usize] as u32),
i8::wrapping_shr(a[30usize], b[30usize] as u32),
i8::wrapping_shr(a[31usize], b[31usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn max_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
_mm256_max_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
_mm256_min_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> mask8x32<Avx2> {
_mm256_cmpeq_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> mask8x32<Avx2> {
_mm256_cmpgt_epi8(b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> mask8x32<Avx2> {
_mm256_cmpeq_epi8(_mm256_min_epi8(a.into(), b.into()), a.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
let lo = _mm256_unpacklo_epi8(a.into(), b.into());
let hi = _mm256_unpackhi_epi8(a.into(), b.into());
_mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
let lo = _mm256_unpacklo_epi8(a.into(), b.into());
let hi = _mm256_unpackhi_epi8(a.into(), b.into());
_mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
a.into(),
_mm256_setr_epi8(
0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10,
12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
),
));
let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
b.into(),
_mm256_setr_epi8(
0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10,
12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
),
));
_mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> i8x32<Avx2> {
let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
a.into(),
_mm256_setr_epi8(
0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10,
12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
),
));
let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
b.into(),
_mm256_setr_epi8(
0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10,
12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
),
));
_mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> (i8x32<Self>, i8x32<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> (i8x32<Avx2>, i8x32<Avx2>) {
let lo = _mm256_unpacklo_epi8(a.into(), b.into());
let hi = _mm256_unpackhi_epi8(a.into(), b.into());
(
_mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token),
_mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> (i8x32<Self>, i8x32<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>, b: i8x32<Avx2>) -> (i8x32<Avx2>, i8x32<Avx2>) {
let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
a.into(),
_mm256_setr_epi8(
0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10,
12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
),
));
let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
b.into(),
_mm256_setr_epi8(
0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10,
12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
),
));
(
_mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token),
_mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_i8x32(self, a: mask8x32<Self>, b: i8x32<Self>, c: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask8x32<Avx2>,
b: i8x32<Avx2>,
c: i8x32<Avx2>,
) -> i8x32<Avx2> {
_mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x64<Self> {
i8x64 {
val: crate::support::Aligned512([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn split_i8x32(self, a: i8x32<Self>) -> (i8x16<Self>, i8x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>) -> (i8x16<Avx2>, i8x16<Avx2>) {
(
_mm256_extracti128_si256::<0>(a.into()).simd_into(token),
_mm256_extracti128_si256::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_i8x32(self, a: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>) -> i8x32<Avx2> {
_mm256_sub_epi8(_mm256_setzero_si256(), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn widen_i8x32(self, a: i8x32<Self>) -> (i16x16<Self>, i16x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i8x32<Avx2>) -> (i16x16<Avx2>, i16x16<Avx2>) {
let raw = a.into();
(
_mm256_cvtepi8_epi16(_mm256_castsi256_si128(raw)).simd_into(token),
_mm256_cvtepi8_epi16(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_u8x32(self, val: u8) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: u8) -> u8x32<Avx2> {
_mm256_set1_epi8(val.cast_signed()).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_u8x32<const SHIFT: usize>(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
if SHIFT >= 32usize {
return b;
}
let result = cross_block_alignr_256x1(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn slide_within_blocks_u8x32<const SHIFT: usize>(
self,
a: u8x32<Self>,
b: u8x32<Self>,
) -> u8x32<Self> {
if SHIFT >= 16usize {
return b;
}
let result = dyn_alignr_256(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, indices: u8x32<Avx2>) -> u8x32<Avx2> {
let result = _mm256_shuffle_epi8(Bytes::to_bytes(a).val.0, indices.into());
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn swizzle_dyn_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, indices: u8x32<Avx2>) -> u8x32<Avx2> {
let bytes = Bytes::to_bytes(a).val.0;
let indices = indices.into();
let swapped = _mm256_permute2x128_si256::<0x01>(bytes, bytes);
let local = _mm256_shuffle_epi8(bytes, indices);
let remote = _mm256_shuffle_epi8(swapped, indices);
let select_remote = _mm256_slli_epi16::<3>(indices);
let flip_high_lane = _mm256_set_m128i(_mm_set1_epi8(i8::MIN), _mm_setzero_si128());
let select_remote = _mm256_xor_si256(select_remote, flip_high_lane);
let result = _mm256_blendv_epi8(local, remote, select_remote);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn swizzle_dyn_precise_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, indices: u8x32<Avx2>) -> u8x32<Avx2> {
let bytes = Bytes::to_bytes(a);
let idxs = indices;
let lolo = _mm256_permute2x128_si256::<0x00>(bytes.val.0, bytes.val.0);
let hihi = _mm256_permute2x128_si256::<0x11>(bytes.val.0, bytes.val.0);
let control = _mm256_adds_epu8(idxs.into(), _mm256_set1_epi8(0x60));
let select_high = _mm256_slli_epi16::<3>(control);
let from_low = _mm256_shuffle_epi8(lolo, control);
let from_high = _mm256_shuffle_epi8(hihi, control);
let result = _mm256_blendv_epi8(from_low, from_high, select_high);
let result_bytes = u8x32 {
val: crate::support::Aligned256(result),
simd: token,
};
Bytes::from_bytes(result_bytes)
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
_mm256_add_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
_mm256_sub_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
let dst_even = _mm256_mullo_epi16(a.into(), b.into());
let dst_odd = _mm256_mullo_epi16(
_mm256_srli_epi16::<8>(a.into()),
_mm256_srli_epi16::<8>(b.into()),
);
_mm256_or_si256(
_mm256_slli_epi16(dst_odd, 8),
_mm256_and_si256(dst_even, _mm256_set1_epi16(0xFF)),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
_mm256_and_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
_mm256_or_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
_mm256_xor_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u8x32(self, a: u8x32<Self>) -> u8x32<Self> {
a ^ !0
}
#[inline(always)]
fn shl_u8x32(self, a: u8x32<Self>, shift: u32) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, shift: u32) -> u8x32<Avx2> {
let val = a.into();
let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
let byte_mask = _mm256_set1_epi8(mask_byte);
_mm256_sll_epi16(_mm256_and_si256(val, byte_mask), shift_count).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
[
u8::wrapping_shl(a[0usize], b[0usize] as u32),
u8::wrapping_shl(a[1usize], b[1usize] as u32),
u8::wrapping_shl(a[2usize], b[2usize] as u32),
u8::wrapping_shl(a[3usize], b[3usize] as u32),
u8::wrapping_shl(a[4usize], b[4usize] as u32),
u8::wrapping_shl(a[5usize], b[5usize] as u32),
u8::wrapping_shl(a[6usize], b[6usize] as u32),
u8::wrapping_shl(a[7usize], b[7usize] as u32),
u8::wrapping_shl(a[8usize], b[8usize] as u32),
u8::wrapping_shl(a[9usize], b[9usize] as u32),
u8::wrapping_shl(a[10usize], b[10usize] as u32),
u8::wrapping_shl(a[11usize], b[11usize] as u32),
u8::wrapping_shl(a[12usize], b[12usize] as u32),
u8::wrapping_shl(a[13usize], b[13usize] as u32),
u8::wrapping_shl(a[14usize], b[14usize] as u32),
u8::wrapping_shl(a[15usize], b[15usize] as u32),
u8::wrapping_shl(a[16usize], b[16usize] as u32),
u8::wrapping_shl(a[17usize], b[17usize] as u32),
u8::wrapping_shl(a[18usize], b[18usize] as u32),
u8::wrapping_shl(a[19usize], b[19usize] as u32),
u8::wrapping_shl(a[20usize], b[20usize] as u32),
u8::wrapping_shl(a[21usize], b[21usize] as u32),
u8::wrapping_shl(a[22usize], b[22usize] as u32),
u8::wrapping_shl(a[23usize], b[23usize] as u32),
u8::wrapping_shl(a[24usize], b[24usize] as u32),
u8::wrapping_shl(a[25usize], b[25usize] as u32),
u8::wrapping_shl(a[26usize], b[26usize] as u32),
u8::wrapping_shl(a[27usize], b[27usize] as u32),
u8::wrapping_shl(a[28usize], b[28usize] as u32),
u8::wrapping_shl(a[29usize], b[29usize] as u32),
u8::wrapping_shl(a[30usize], b[30usize] as u32),
u8::wrapping_shl(a[31usize], b[31usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn shr_u8x32(self, a: u8x32<Self>, shift: u32) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, shift: u32) -> u8x32<Avx2> {
let val = a.into();
let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
let mask_byte = 0xff_u32.wrapping_shr(shift) as i8;
let byte_mask = _mm256_set1_epi8(mask_byte);
let shifted = _mm256_srl_epi16(val, shift_count);
let result = { _mm256_and_si256(shifted, byte_mask) };
result.simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
[
u8::wrapping_shr(a[0usize], b[0usize] as u32),
u8::wrapping_shr(a[1usize], b[1usize] as u32),
u8::wrapping_shr(a[2usize], b[2usize] as u32),
u8::wrapping_shr(a[3usize], b[3usize] as u32),
u8::wrapping_shr(a[4usize], b[4usize] as u32),
u8::wrapping_shr(a[5usize], b[5usize] as u32),
u8::wrapping_shr(a[6usize], b[6usize] as u32),
u8::wrapping_shr(a[7usize], b[7usize] as u32),
u8::wrapping_shr(a[8usize], b[8usize] as u32),
u8::wrapping_shr(a[9usize], b[9usize] as u32),
u8::wrapping_shr(a[10usize], b[10usize] as u32),
u8::wrapping_shr(a[11usize], b[11usize] as u32),
u8::wrapping_shr(a[12usize], b[12usize] as u32),
u8::wrapping_shr(a[13usize], b[13usize] as u32),
u8::wrapping_shr(a[14usize], b[14usize] as u32),
u8::wrapping_shr(a[15usize], b[15usize] as u32),
u8::wrapping_shr(a[16usize], b[16usize] as u32),
u8::wrapping_shr(a[17usize], b[17usize] as u32),
u8::wrapping_shr(a[18usize], b[18usize] as u32),
u8::wrapping_shr(a[19usize], b[19usize] as u32),
u8::wrapping_shr(a[20usize], b[20usize] as u32),
u8::wrapping_shr(a[21usize], b[21usize] as u32),
u8::wrapping_shr(a[22usize], b[22usize] as u32),
u8::wrapping_shr(a[23usize], b[23usize] as u32),
u8::wrapping_shr(a[24usize], b[24usize] as u32),
u8::wrapping_shr(a[25usize], b[25usize] as u32),
u8::wrapping_shr(a[26usize], b[26usize] as u32),
u8::wrapping_shr(a[27usize], b[27usize] as u32),
u8::wrapping_shr(a[28usize], b[28usize] as u32),
u8::wrapping_shr(a[29usize], b[29usize] as u32),
u8::wrapping_shr(a[30usize], b[30usize] as u32),
u8::wrapping_shr(a[31usize], b[31usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn max_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
_mm256_max_epu8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
_mm256_min_epu8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> mask8x32<Avx2> {
_mm256_cmpeq_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> mask8x32<Avx2> {
{
let sign_bit = _mm256_set1_epi8(0x80u8.cast_signed());
let lhs_signed = _mm256_xor_si256(b.into(), sign_bit);
let rhs_signed = _mm256_xor_si256(a.into(), sign_bit);
_mm256_cmpgt_epi8(lhs_signed, rhs_signed)
}
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> mask8x32<Avx2> {
_mm256_cmpeq_epi8(_mm256_min_epu8(a.into(), b.into()), a.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
let lo = _mm256_unpacklo_epi8(a.into(), b.into());
let hi = _mm256_unpackhi_epi8(a.into(), b.into());
_mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
let lo = _mm256_unpacklo_epi8(a.into(), b.into());
let hi = _mm256_unpackhi_epi8(a.into(), b.into());
_mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
a.into(),
_mm256_setr_epi8(
0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10,
12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
),
));
let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
b.into(),
_mm256_setr_epi8(
0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10,
12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
),
));
_mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> u8x32<Avx2> {
let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
a.into(),
_mm256_setr_epi8(
0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10,
12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
),
));
let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
b.into(),
_mm256_setr_epi8(
0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10,
12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
),
));
_mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> (u8x32<Self>, u8x32<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> (u8x32<Avx2>, u8x32<Avx2>) {
let lo = _mm256_unpacklo_epi8(a.into(), b.into());
let hi = _mm256_unpackhi_epi8(a.into(), b.into());
(
_mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token),
_mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> (u8x32<Self>, u8x32<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>, b: u8x32<Avx2>) -> (u8x32<Avx2>, u8x32<Avx2>) {
let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
a.into(),
_mm256_setr_epi8(
0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10,
12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
),
));
let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
b.into(),
_mm256_setr_epi8(
0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10,
12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
),
));
(
_mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token),
_mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_u8x32(self, a: mask8x32<Self>, b: u8x32<Self>, c: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask8x32<Avx2>,
b: u8x32<Avx2>,
c: u8x32<Avx2>,
) -> u8x32<Avx2> {
_mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x64<Self> {
u8x64 {
val: crate::support::Aligned512([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn split_u8x32(self, a: u8x32<Self>) -> (u8x16<Self>, u8x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>) -> (u8x16<Avx2>, u8x16<Avx2>) {
(
_mm256_extracti128_si256::<0>(a.into()).simd_into(token),
_mm256_extracti128_si256::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn widen_u8x32(self, a: u8x32<Self>) -> (u16x16<Self>, u16x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x32<Avx2>) -> (u16x16<Avx2>, u16x16<Avx2>) {
let raw = a.into();
(
_mm256_cvtepu8_epi16(_mm256_castsi256_si128(raw)).simd_into(token),
_mm256_cvtepu8_epi16(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_mask8x32(self, val: bool) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: bool) -> mask8x32<Avx2> {
let val: i8 = if val { !0 } else { 0 };
_mm256_set1_epi8(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn from_bitmask_mask8x32(self, bits: u64) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, bits: u64) -> mask8x32<Avx2> {
{
let bit_bytes = _mm256_broadcastsi128_si256(_mm_cvtsi32_si128(bits as i32));
let bit_bytes = _mm256_shuffle_epi8(
bit_bytes,
_mm256_setr_epi8(
0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2,
3, 3, 3, 3, 3, 3, 3, 3,
),
);
let bit_mask = _mm256_setr_epi8(
1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16,
32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128,
);
_mm256_cmpeq_epi8(_mm256_and_si256(bit_bytes, bit_mask), bit_mask)
}
.simd_into(token)
}
);
kernel(self, bits)
}
#[inline(always)]
fn to_bitmask_mask8x32(self, a: mask8x32<Self>) -> u64 {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask8x32<Avx2>) -> u64 {
_mm256_movemask_epi8(a.into()) as u32 as u64
}
);
kernel(self, a)
}
#[inline(always)]
fn set_mask8x32(self, a: &mut mask8x32<Self>, index: usize, value: bool) -> () {
assert!(
index < 32usize,
"mask lane index {index} is out of bounds for {} lanes",
32usize
);
let mut lanes: [i8; 32usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn and_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask8x32<Avx2>, b: mask8x32<Avx2>) -> mask8x32<Avx2> {
_mm256_and_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask8x32<Avx2>, b: mask8x32<Avx2>) -> mask8x32<Avx2> {
_mm256_or_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask8x32<Avx2>, b: mask8x32<Avx2>) -> mask8x32<Avx2> {
_mm256_xor_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_mask8x32(self, a: mask8x32<Self>) -> mask8x32<Self> {
self.xor_mask8x32(a, self.splat_mask8x32(true))
}
#[inline(always)]
fn select_mask8x32(
self,
a: mask8x32<Self>,
b: mask8x32<Self>,
c: mask8x32<Self>,
) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask8x32<Avx2>,
b: mask8x32<Avx2>,
c: mask8x32<Avx2>,
) -> mask8x32<Avx2> {
_mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn simd_eq_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask8x32<Avx2>, b: mask8x32<Avx2>) -> mask8x32<Avx2> {
_mm256_cmpeq_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn any_true_mask8x32(self, a: mask8x32<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask8x32<Avx2>) -> bool {
_mm256_movemask_epi8(a.into()) as u32 != 0
}
);
kernel(self, a)
}
#[inline(always)]
fn all_true_mask8x32(self, a: mask8x32<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask8x32<Avx2>) -> bool {
_mm256_movemask_epi8(a.into()) as u32 == 0xffffffff
}
);
kernel(self, a)
}
#[inline(always)]
fn any_false_mask8x32(self, a: mask8x32<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask8x32<Avx2>) -> bool {
_mm256_movemask_epi8(a.into()) as u32 != 0xffffffff
}
);
kernel(self, a)
}
#[inline(always)]
fn all_false_mask8x32(self, a: mask8x32<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask8x32<Avx2>) -> bool {
_mm256_movemask_epi8(a.into()) as u32 == 0
}
);
kernel(self, a)
}
#[inline(always)]
fn combine_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x64<Self> {
mask8x64 {
val: crate::support::Aligned512([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn split_mask8x32(self, a: mask8x32<Self>) -> (mask8x16<Self>, mask8x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask8x32<Avx2>) -> (mask8x16<Avx2>, mask8x16<Avx2>) {
(
_mm256_extracti128_si256::<0>(a.into()).simd_into(token),
_mm256_extracti128_si256::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_i16x16(self, val: i16) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: i16) -> i16x16<Avx2> {
_mm256_set1_epi16(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_i16x16<const SHIFT: usize>(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = cross_block_alignr_256x1(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 2usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn slide_within_blocks_i16x16<const SHIFT: usize>(
self,
a: i16x16<Self>,
b: i16x16<Self>,
) -> i16x16<Self> {
if SHIFT >= 8usize {
return b;
}
let result = dyn_alignr_256(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 2usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn add_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
_mm256_add_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
_mm256_sub_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
_mm256_mullo_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
_mm256_and_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
_mm256_or_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
_mm256_xor_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i16x16(self, a: i16x16<Self>) -> i16x16<Self> {
a ^ !0
}
#[inline(always)]
fn shl_i16x16(self, a: i16x16<Self>, shift: u32) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>, shift: u32) -> i16x16<Avx2> {
_mm256_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
[
i16::wrapping_shl(a[0usize], b[0usize] as u32),
i16::wrapping_shl(a[1usize], b[1usize] as u32),
i16::wrapping_shl(a[2usize], b[2usize] as u32),
i16::wrapping_shl(a[3usize], b[3usize] as u32),
i16::wrapping_shl(a[4usize], b[4usize] as u32),
i16::wrapping_shl(a[5usize], b[5usize] as u32),
i16::wrapping_shl(a[6usize], b[6usize] as u32),
i16::wrapping_shl(a[7usize], b[7usize] as u32),
i16::wrapping_shl(a[8usize], b[8usize] as u32),
i16::wrapping_shl(a[9usize], b[9usize] as u32),
i16::wrapping_shl(a[10usize], b[10usize] as u32),
i16::wrapping_shl(a[11usize], b[11usize] as u32),
i16::wrapping_shl(a[12usize], b[12usize] as u32),
i16::wrapping_shl(a[13usize], b[13usize] as u32),
i16::wrapping_shl(a[14usize], b[14usize] as u32),
i16::wrapping_shl(a[15usize], b[15usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn shr_i16x16(self, a: i16x16<Self>, shift: u32) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>, shift: u32) -> i16x16<Avx2> {
_mm256_sra_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
[
i16::wrapping_shr(a[0usize], b[0usize] as u32),
i16::wrapping_shr(a[1usize], b[1usize] as u32),
i16::wrapping_shr(a[2usize], b[2usize] as u32),
i16::wrapping_shr(a[3usize], b[3usize] as u32),
i16::wrapping_shr(a[4usize], b[4usize] as u32),
i16::wrapping_shr(a[5usize], b[5usize] as u32),
i16::wrapping_shr(a[6usize], b[6usize] as u32),
i16::wrapping_shr(a[7usize], b[7usize] as u32),
i16::wrapping_shr(a[8usize], b[8usize] as u32),
i16::wrapping_shr(a[9usize], b[9usize] as u32),
i16::wrapping_shr(a[10usize], b[10usize] as u32),
i16::wrapping_shr(a[11usize], b[11usize] as u32),
i16::wrapping_shr(a[12usize], b[12usize] as u32),
i16::wrapping_shr(a[13usize], b[13usize] as u32),
i16::wrapping_shr(a[14usize], b[14usize] as u32),
i16::wrapping_shr(a[15usize], b[15usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn max_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
_mm256_max_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
_mm256_min_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> mask16x16<Avx2> {
_mm256_cmpeq_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> mask16x16<Avx2> {
_mm256_cmpgt_epi16(b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> mask16x16<Avx2> {
_mm256_cmpeq_epi16(_mm256_min_epi16(a.into(), b.into()), a.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
let lo = _mm256_unpacklo_epi16(a.into(), b.into());
let hi = _mm256_unpackhi_epi16(a.into(), b.into());
_mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
let lo = _mm256_unpacklo_epi16(a.into(), b.into());
let hi = _mm256_unpackhi_epi16(a.into(), b.into());
_mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
a.into(),
_mm256_setr_epi8(
0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12,
13, 2, 3, 6, 7, 10, 11, 14, 15,
),
));
let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
b.into(),
_mm256_setr_epi8(
0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12,
13, 2, 3, 6, 7, 10, 11, 14, 15,
),
));
_mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i16x16<Avx2> {
let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
a.into(),
_mm256_setr_epi8(
0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12,
13, 2, 3, 6, 7, 10, 11, 14, 15,
),
));
let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
b.into(),
_mm256_setr_epi8(
0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12,
13, 2, 3, 6, 7, 10, 11, 14, 15,
),
));
_mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> (i16x16<Self>, i16x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: i16x16<Avx2>,
b: i16x16<Avx2>,
) -> (i16x16<Avx2>, i16x16<Avx2>) {
let lo = _mm256_unpacklo_epi16(a.into(), b.into());
let hi = _mm256_unpackhi_epi16(a.into(), b.into());
(
_mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token),
_mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> (i16x16<Self>, i16x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: i16x16<Avx2>,
b: i16x16<Avx2>,
) -> (i16x16<Avx2>, i16x16<Avx2>) {
let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
a.into(),
_mm256_setr_epi8(
0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12,
13, 2, 3, 6, 7, 10, 11, 14, 15,
),
));
let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
b.into(),
_mm256_setr_epi8(
0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12,
13, 2, 3, 6, 7, 10, 11, 14, 15,
),
));
(
_mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token),
_mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_i16x16(self, a: mask16x16<Self>, b: i16x16<Self>, c: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask16x16<Avx2>,
b: i16x16<Avx2>,
c: i16x16<Avx2>,
) -> i16x16<Avx2> {
_mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x32<Self> {
i16x32 {
val: crate::support::Aligned512([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn split_i16x16(self, a: i16x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>) -> (i16x8<Avx2>, i16x8<Avx2>) {
(
_mm256_extracti128_si256::<0>(a.into()).simd_into(token),
_mm256_extracti128_si256::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_i16x16(self, a: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>) -> i16x16<Avx2> {
_mm256_sub_epi16(_mm256_setzero_si256(), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn widen_i16x16(self, a: i16x16<Self>) -> (i32x8<Self>, i32x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>) -> (i32x8<Avx2>, i32x8<Avx2>) {
let raw = a.into();
(
_mm256_cvtepi16_epi32(_mm256_castsi256_si128(raw)).simd_into(token),
_mm256_cvtepi16_epi32(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn narrow_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i8x32<Avx2> {
_mm256_permute4x64_epi64::<0xd8>({
let mask = _mm256_set1_epi16(0xff);
_mm256_packus_epi16(
_mm256_and_si256(a.into(), mask),
_mm256_and_si256(b.into(), mask),
)
})
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i16x16<Avx2>, b: i16x16<Avx2>) -> i8x32<Avx2> {
_mm256_permute4x64_epi64::<0xd8>(_mm256_packs_epi16(a.into(), b.into()))
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn relaxed_narrow_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i8x32<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= i8::MIN as i16 && value <= i8::MAX as i16 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.saturating_narrow_i16x16(a, b)
}
#[inline(always)]
fn splat_u16x16(self, val: u16) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: u16) -> u16x16<Avx2> {
_mm256_set1_epi16(val.cast_signed()).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_u16x16<const SHIFT: usize>(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = cross_block_alignr_256x1(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 2usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn slide_within_blocks_u16x16<const SHIFT: usize>(
self,
a: u16x16<Self>,
b: u16x16<Self>,
) -> u16x16<Self> {
if SHIFT >= 8usize {
return b;
}
let result = dyn_alignr_256(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 2usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn add_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
_mm256_add_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
_mm256_sub_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
_mm256_mullo_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
_mm256_and_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
_mm256_or_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
_mm256_xor_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u16x16(self, a: u16x16<Self>) -> u16x16<Self> {
a ^ !0
}
#[inline(always)]
fn shl_u16x16(self, a: u16x16<Self>, shift: u32) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>, shift: u32) -> u16x16<Avx2> {
_mm256_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
[
u16::wrapping_shl(a[0usize], b[0usize] as u32),
u16::wrapping_shl(a[1usize], b[1usize] as u32),
u16::wrapping_shl(a[2usize], b[2usize] as u32),
u16::wrapping_shl(a[3usize], b[3usize] as u32),
u16::wrapping_shl(a[4usize], b[4usize] as u32),
u16::wrapping_shl(a[5usize], b[5usize] as u32),
u16::wrapping_shl(a[6usize], b[6usize] as u32),
u16::wrapping_shl(a[7usize], b[7usize] as u32),
u16::wrapping_shl(a[8usize], b[8usize] as u32),
u16::wrapping_shl(a[9usize], b[9usize] as u32),
u16::wrapping_shl(a[10usize], b[10usize] as u32),
u16::wrapping_shl(a[11usize], b[11usize] as u32),
u16::wrapping_shl(a[12usize], b[12usize] as u32),
u16::wrapping_shl(a[13usize], b[13usize] as u32),
u16::wrapping_shl(a[14usize], b[14usize] as u32),
u16::wrapping_shl(a[15usize], b[15usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn shr_u16x16(self, a: u16x16<Self>, shift: u32) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>, shift: u32) -> u16x16<Avx2> {
_mm256_srl_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
[
u16::wrapping_shr(a[0usize], b[0usize] as u32),
u16::wrapping_shr(a[1usize], b[1usize] as u32),
u16::wrapping_shr(a[2usize], b[2usize] as u32),
u16::wrapping_shr(a[3usize], b[3usize] as u32),
u16::wrapping_shr(a[4usize], b[4usize] as u32),
u16::wrapping_shr(a[5usize], b[5usize] as u32),
u16::wrapping_shr(a[6usize], b[6usize] as u32),
u16::wrapping_shr(a[7usize], b[7usize] as u32),
u16::wrapping_shr(a[8usize], b[8usize] as u32),
u16::wrapping_shr(a[9usize], b[9usize] as u32),
u16::wrapping_shr(a[10usize], b[10usize] as u32),
u16::wrapping_shr(a[11usize], b[11usize] as u32),
u16::wrapping_shr(a[12usize], b[12usize] as u32),
u16::wrapping_shr(a[13usize], b[13usize] as u32),
u16::wrapping_shr(a[14usize], b[14usize] as u32),
u16::wrapping_shr(a[15usize], b[15usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn max_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
_mm256_max_epu16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
_mm256_min_epu16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> mask16x16<Avx2> {
_mm256_cmpeq_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> mask16x16<Avx2> {
{
let sign_bit = _mm256_set1_epi16(0x8000u16.cast_signed());
let lhs_signed = _mm256_xor_si256(b.into(), sign_bit);
let rhs_signed = _mm256_xor_si256(a.into(), sign_bit);
_mm256_cmpgt_epi16(lhs_signed, rhs_signed)
}
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> mask16x16<Avx2> {
_mm256_cmpeq_epi16(_mm256_min_epu16(a.into(), b.into()), a.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
let lo = _mm256_unpacklo_epi16(a.into(), b.into());
let hi = _mm256_unpackhi_epi16(a.into(), b.into());
_mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
let lo = _mm256_unpacklo_epi16(a.into(), b.into());
let hi = _mm256_unpackhi_epi16(a.into(), b.into());
_mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
a.into(),
_mm256_setr_epi8(
0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12,
13, 2, 3, 6, 7, 10, 11, 14, 15,
),
));
let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
b.into(),
_mm256_setr_epi8(
0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12,
13, 2, 3, 6, 7, 10, 11, 14, 15,
),
));
_mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u16x16<Avx2> {
let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
a.into(),
_mm256_setr_epi8(
0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12,
13, 2, 3, 6, 7, 10, 11, 14, 15,
),
));
let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
b.into(),
_mm256_setr_epi8(
0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12,
13, 2, 3, 6, 7, 10, 11, 14, 15,
),
));
_mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> (u16x16<Self>, u16x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: u16x16<Avx2>,
b: u16x16<Avx2>,
) -> (u16x16<Avx2>, u16x16<Avx2>) {
let lo = _mm256_unpacklo_epi16(a.into(), b.into());
let hi = _mm256_unpackhi_epi16(a.into(), b.into());
(
_mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token),
_mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> (u16x16<Self>, u16x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: u16x16<Avx2>,
b: u16x16<Avx2>,
) -> (u16x16<Avx2>, u16x16<Avx2>) {
let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
a.into(),
_mm256_setr_epi8(
0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12,
13, 2, 3, 6, 7, 10, 11, 14, 15,
),
));
let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(_mm256_shuffle_epi8(
b.into(),
_mm256_setr_epi8(
0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, 0, 1, 4, 5, 8, 9, 12,
13, 2, 3, 6, 7, 10, 11, 14, 15,
),
));
(
_mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token),
_mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_u16x16(self, a: mask16x16<Self>, b: u16x16<Self>, c: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask16x16<Avx2>,
b: u16x16<Avx2>,
c: u16x16<Avx2>,
) -> u16x16<Avx2> {
_mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x32<Self> {
u16x32 {
val: crate::support::Aligned512([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn split_u16x16(self, a: u16x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>) -> (u16x8<Avx2>, u16x8<Avx2>) {
(
_mm256_extracti128_si256::<0>(a.into()).simd_into(token),
_mm256_extracti128_si256::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn widen_u16x16(self, a: u16x16<Self>) -> (u32x8<Self>, u32x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>) -> (u32x8<Avx2>, u32x8<Avx2>) {
let raw = a.into();
(
_mm256_cvtepu16_epi32(_mm256_castsi256_si128(raw)).simd_into(token),
_mm256_cvtepu16_epi32(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn narrow_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u8x32<Avx2> {
_mm256_permute4x64_epi64::<0xd8>({
let mask = _mm256_set1_epi16(0xff);
_mm256_packus_epi16(
_mm256_and_si256(a.into(), mask),
_mm256_and_si256(b.into(), mask),
)
})
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u16x16<Avx2>, b: u16x16<Avx2>) -> u8x32<Avx2> {
_mm256_permute4x64_epi64::<0xd8>({
let max = _mm256_set1_epi16(0xff);
_mm256_packus_epi16(
_mm256_min_epu16(a.into(), max),
_mm256_min_epu16(b.into(), max),
)
})
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn relaxed_narrow_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u8x32<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= u8::MIN as u16 && value <= u8::MAX as u16 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.narrow_u16x16(a, b)
}
#[inline(always)]
fn splat_mask16x16(self, val: bool) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: bool) -> mask16x16<Avx2> {
let val: i16 = if val { !0 } else { 0 };
_mm256_set1_epi16(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn from_bitmask_mask16x16(self, bits: u64) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, bits: u64) -> mask16x16<Avx2> {
{
let bit_lanes = _mm256_set1_epi16(bits as i16);
let bit_mask = _mm256_setr_epi16(
1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384,
-32768,
);
_mm256_cmpeq_epi16(_mm256_and_si256(bit_lanes, bit_mask), bit_mask)
}
.simd_into(token)
}
);
kernel(self, bits)
}
#[inline(always)]
fn to_bitmask_mask16x16(self, a: mask16x16<Self>) -> u64 {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x16<Avx2>) -> u64 {
{
let halves: [__m128i; 2usize] =
crate::transmute::checked_transmute_copy(&a.val.0);
let packed = _mm_packs_epi16(halves[0], halves[1]);
_mm_movemask_epi8(packed) as u32 as u64
}
}
);
kernel(self, a)
}
#[inline(always)]
fn set_mask16x16(self, a: &mut mask16x16<Self>, index: usize, value: bool) -> () {
assert!(
index < 16usize,
"mask lane index {index} is out of bounds for {} lanes",
16usize
);
let mut lanes: [i16; 16usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn and_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x16<Avx2>, b: mask16x16<Avx2>) -> mask16x16<Avx2> {
_mm256_and_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x16<Avx2>, b: mask16x16<Avx2>) -> mask16x16<Avx2> {
_mm256_or_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x16<Avx2>, b: mask16x16<Avx2>) -> mask16x16<Avx2> {
_mm256_xor_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_mask16x16(self, a: mask16x16<Self>) -> mask16x16<Self> {
self.xor_mask16x16(a, self.splat_mask16x16(true))
}
#[inline(always)]
fn select_mask16x16(
self,
a: mask16x16<Self>,
b: mask16x16<Self>,
c: mask16x16<Self>,
) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask16x16<Avx2>,
b: mask16x16<Avx2>,
c: mask16x16<Avx2>,
) -> mask16x16<Avx2> {
_mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn simd_eq_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x16<Avx2>, b: mask16x16<Avx2>) -> mask16x16<Avx2> {
_mm256_cmpeq_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn any_true_mask16x16(self, a: mask16x16<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x16<Avx2>) -> bool {
_mm256_movemask_epi8(a.into()) as u32 != 0
}
);
kernel(self, a)
}
#[inline(always)]
fn all_true_mask16x16(self, a: mask16x16<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x16<Avx2>) -> bool {
_mm256_movemask_epi8(a.into()) as u32 == 0xffffffff
}
);
kernel(self, a)
}
#[inline(always)]
fn any_false_mask16x16(self, a: mask16x16<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x16<Avx2>) -> bool {
_mm256_movemask_epi8(a.into()) as u32 != 0xffffffff
}
);
kernel(self, a)
}
#[inline(always)]
fn all_false_mask16x16(self, a: mask16x16<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x16<Avx2>) -> bool {
_mm256_movemask_epi8(a.into()) as u32 == 0
}
);
kernel(self, a)
}
#[inline(always)]
fn combine_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x32<Self> {
mask16x32 {
val: crate::support::Aligned512([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn split_mask16x16(self, a: mask16x16<Self>) -> (mask16x8<Self>, mask16x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x16<Avx2>) -> (mask16x8<Avx2>, mask16x8<Avx2>) {
(
_mm256_extracti128_si256::<0>(a.into()).simd_into(token),
_mm256_extracti128_si256::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_i32x8(self, val: i32) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: i32) -> i32x8<Avx2> {
_mm256_set1_epi32(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_i32x8<const SHIFT: usize>(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = cross_block_alignr_256x1(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn slide_within_blocks_i32x8<const SHIFT: usize>(
self,
a: i32x8<Self>,
b: i32x8<Self>,
) -> i32x8<Self> {
if SHIFT >= 4usize {
return b;
}
let result = dyn_alignr_256(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn add_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
_mm256_add_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
_mm256_sub_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
_mm256_mullo_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
_mm256_and_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
_mm256_or_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
_mm256_xor_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i32x8(self, a: i32x8<Self>) -> i32x8<Self> {
a ^ !0
}
#[inline(always)]
fn shl_i32x8(self, a: i32x8<Self>, shift: u32) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, shift: u32) -> i32x8<Avx2> {
_mm256_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
_mm256_sllv_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_i32x8(self, a: i32x8<Self>, shift: u32) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, shift: u32) -> i32x8<Avx2> {
_mm256_sra_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
_mm256_srav_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
_mm256_max_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
_mm256_min_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> mask32x8<Avx2> {
_mm256_cmpeq_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> mask32x8<Avx2> {
_mm256_cmpgt_epi32(b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> mask32x8<Avx2> {
_mm256_cmpeq_epi32(_mm256_min_epi32(a.into(), b.into()), a.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
let lo = _mm256_unpacklo_epi32(a.into(), b.into());
let hi = _mm256_unpackhi_epi32(a.into(), b.into());
_mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
let lo = _mm256_unpacklo_epi32(a.into(), b.into());
let hi = _mm256_unpackhi_epi32(a.into(), b.into());
_mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
let t1 = _mm256_permutevar8x32_epi32(
a.into(),
_mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
);
let t2 = _mm256_permutevar8x32_epi32(
b.into(),
_mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
);
_mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i32x8<Avx2> {
let t1 = _mm256_permutevar8x32_epi32(
a.into(),
_mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
);
let t2 = _mm256_permutevar8x32_epi32(
b.into(),
_mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
);
_mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> (i32x8<Self>, i32x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> (i32x8<Avx2>, i32x8<Avx2>) {
let lo = _mm256_unpacklo_epi32(a.into(), b.into());
let hi = _mm256_unpackhi_epi32(a.into(), b.into());
(
_mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token),
_mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> (i32x8<Self>, i32x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> (i32x8<Avx2>, i32x8<Avx2>) {
let t1 = _mm256_permutevar8x32_epi32(
a.into(),
_mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
);
let t2 = _mm256_permutevar8x32_epi32(
b.into(),
_mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
);
(
_mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token),
_mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_i32x8(self, a: mask32x8<Self>, b: i32x8<Self>, c: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask32x8<Avx2>,
b: i32x8<Avx2>,
c: i32x8<Avx2>,
) -> i32x8<Avx2> {
_mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x16<Self> {
i32x16 {
val: crate::support::Aligned512([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn split_i32x8(self, a: i32x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>) -> (i32x4<Avx2>, i32x4<Avx2>) {
(
_mm256_extracti128_si256::<0>(a.into()).simd_into(token),
_mm256_extracti128_si256::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_i32x8(self, a: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>) -> i32x8<Avx2> {
_mm256_sub_epi32(_mm256_setzero_si256(), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn widen_i32x8(self, a: i32x8<Self>) -> (i64x4<Self>, i64x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>) -> (i64x4<Avx2>, i64x4<Avx2>) {
let raw = a.into();
(
_mm256_cvtepi32_epi64(_mm256_castsi256_si128(raw)).simd_into(token),
_mm256_cvtepi32_epi64(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn narrow_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i16x16<Avx2> {
_mm256_permute4x64_epi64::<0xd8>({
let mask = _mm256_set1_epi32(0xffff);
_mm256_packus_epi32(
_mm256_and_si256(a.into(), mask),
_mm256_and_si256(b.into(), mask),
)
})
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>, b: i32x8<Avx2>) -> i16x16<Avx2> {
_mm256_permute4x64_epi64::<0xd8>(_mm256_packs_epi32(a.into(), b.into()))
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn relaxed_narrow_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i16x16<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= i16::MIN as i32 && value <= i16::MAX as i32 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.saturating_narrow_i32x8(a, b)
}
#[inline(always)]
fn cvt_f32_i32x8(self, a: i32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i32x8<Avx2>) -> f32x8<Avx2> {
_mm256_cvtepi32_ps(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_u32x8(self, val: u32) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: u32) -> u32x8<Avx2> {
_mm256_set1_epi32(val.cast_signed()).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_u32x8<const SHIFT: usize>(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = cross_block_alignr_256x1(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn slide_within_blocks_u32x8<const SHIFT: usize>(
self,
a: u32x8<Self>,
b: u32x8<Self>,
) -> u32x8<Self> {
if SHIFT >= 4usize {
return b;
}
let result = dyn_alignr_256(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn add_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
_mm256_add_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
_mm256_sub_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
_mm256_mullo_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
_mm256_and_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
_mm256_or_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
_mm256_xor_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u32x8(self, a: u32x8<Self>) -> u32x8<Self> {
a ^ !0
}
#[inline(always)]
fn shl_u32x8(self, a: u32x8<Self>, shift: u32) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, shift: u32) -> u32x8<Avx2> {
_mm256_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
_mm256_sllv_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_u32x8(self, a: u32x8<Self>, shift: u32) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, shift: u32) -> u32x8<Avx2> {
_mm256_srl_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
_mm256_srlv_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
_mm256_max_epu32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
_mm256_min_epu32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> mask32x8<Avx2> {
_mm256_cmpeq_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> mask32x8<Avx2> {
{
let sign_bit = _mm256_set1_epi32(0x80000000u32.cast_signed());
let lhs_signed = _mm256_xor_si256(b.into(), sign_bit);
let rhs_signed = _mm256_xor_si256(a.into(), sign_bit);
_mm256_cmpgt_epi32(lhs_signed, rhs_signed)
}
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> mask32x8<Avx2> {
_mm256_cmpeq_epi32(_mm256_min_epu32(a.into(), b.into()), a.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
let lo = _mm256_unpacklo_epi32(a.into(), b.into());
let hi = _mm256_unpackhi_epi32(a.into(), b.into());
_mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
let lo = _mm256_unpacklo_epi32(a.into(), b.into());
let hi = _mm256_unpackhi_epi32(a.into(), b.into());
_mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
let t1 = _mm256_permutevar8x32_epi32(
a.into(),
_mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
);
let t2 = _mm256_permutevar8x32_epi32(
b.into(),
_mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
);
_mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u32x8<Avx2> {
let t1 = _mm256_permutevar8x32_epi32(
a.into(),
_mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
);
let t2 = _mm256_permutevar8x32_epi32(
b.into(),
_mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
);
_mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> (u32x8<Self>, u32x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> (u32x8<Avx2>, u32x8<Avx2>) {
let lo = _mm256_unpacklo_epi32(a.into(), b.into());
let hi = _mm256_unpackhi_epi32(a.into(), b.into());
(
_mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token),
_mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> (u32x8<Self>, u32x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> (u32x8<Avx2>, u32x8<Avx2>) {
let t1 = _mm256_permutevar8x32_epi32(
a.into(),
_mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
);
let t2 = _mm256_permutevar8x32_epi32(
b.into(),
_mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7),
);
(
_mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token),
_mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_u32x8(self, a: mask32x8<Self>, b: u32x8<Self>, c: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask32x8<Avx2>,
b: u32x8<Avx2>,
c: u32x8<Avx2>,
) -> u32x8<Avx2> {
_mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x16<Self> {
u32x16 {
val: crate::support::Aligned512([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn split_u32x8(self, a: u32x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>) -> (u32x4<Avx2>, u32x4<Avx2>) {
(
_mm256_extracti128_si256::<0>(a.into()).simd_into(token),
_mm256_extracti128_si256::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn widen_u32x8(self, a: u32x8<Self>) -> (u64x4<Self>, u64x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>) -> (u64x4<Avx2>, u64x4<Avx2>) {
let raw = a.into();
(
_mm256_cvtepu32_epi64(_mm256_castsi256_si128(raw)).simd_into(token),
_mm256_cvtepu32_epi64(_mm256_extracti128_si256::<1>(raw)).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn narrow_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u16x16<Avx2> {
_mm256_permute4x64_epi64::<0xd8>({
let mask = _mm256_set1_epi32(0xffff);
_mm256_packus_epi32(
_mm256_and_si256(a.into(), mask),
_mm256_and_si256(b.into(), mask),
)
})
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>, b: u32x8<Avx2>) -> u16x16<Avx2> {
_mm256_permute4x64_epi64::<0xd8>({
let max = _mm256_set1_epi32(0xffff);
_mm256_packus_epi32(
_mm256_min_epu32(a.into(), max),
_mm256_min_epu32(b.into(), max),
)
})
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn relaxed_narrow_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u16x16<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= u16::MIN as u32 && value <= u16::MAX as u32 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.narrow_u32x8(a, b)
}
#[inline(always)]
fn cvt_f32_u32x8(self, a: u32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u32x8<Avx2>) -> f32x8<Avx2> {
let a = a.into();
let lo = _mm256_blend_epi16::<0xAA>(a, _mm256_set1_epi32(0x4B000000));
let hi = _mm256_blend_epi16::<0xAA>(
_mm256_srli_epi32::<16>(a),
_mm256_set1_epi32(0x53000000),
);
let fhi = _mm256_sub_ps(
_mm256_castsi256_ps(hi),
_mm256_set1_ps(f32::from_bits(0x53000080)),
);
let result = _mm256_add_ps(_mm256_castsi256_ps(lo), fhi);
result.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_mask32x8(self, val: bool) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: bool) -> mask32x8<Avx2> {
let val: i32 = if val { !0 } else { 0 };
_mm256_set1_epi32(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn from_bitmask_mask32x8(self, bits: u64) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, bits: u64) -> mask32x8<Avx2> {
{
let bit_lanes = _mm256_set1_epi32(bits as i32);
let bit_mask = _mm256_setr_epi32(1, 2, 4, 8, 16, 32, 64, 128);
_mm256_cmpeq_epi32(_mm256_and_si256(bit_lanes, bit_mask), bit_mask)
}
.simd_into(token)
}
);
kernel(self, bits)
}
#[inline(always)]
fn to_bitmask_mask32x8(self, a: mask32x8<Self>) -> u64 {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask32x8<Avx2>) -> u64 {
_mm256_movemask_ps(_mm256_castsi256_ps(a.into())) as u32 as u64
}
);
kernel(self, a)
}
#[inline(always)]
fn set_mask32x8(self, a: &mut mask32x8<Self>, index: usize, value: bool) -> () {
assert!(
index < 8usize,
"mask lane index {index} is out of bounds for {} lanes",
8usize
);
let mut lanes: [i32; 8usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn and_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask32x8<Avx2>, b: mask32x8<Avx2>) -> mask32x8<Avx2> {
_mm256_and_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask32x8<Avx2>, b: mask32x8<Avx2>) -> mask32x8<Avx2> {
_mm256_or_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask32x8<Avx2>, b: mask32x8<Avx2>) -> mask32x8<Avx2> {
_mm256_xor_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_mask32x8(self, a: mask32x8<Self>) -> mask32x8<Self> {
self.xor_mask32x8(a, self.splat_mask32x8(true))
}
#[inline(always)]
fn select_mask32x8(
self,
a: mask32x8<Self>,
b: mask32x8<Self>,
c: mask32x8<Self>,
) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask32x8<Avx2>,
b: mask32x8<Avx2>,
c: mask32x8<Avx2>,
) -> mask32x8<Avx2> {
_mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn simd_eq_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask32x8<Avx2>, b: mask32x8<Avx2>) -> mask32x8<Avx2> {
_mm256_cmpeq_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn any_true_mask32x8(self, a: mask32x8<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask32x8<Avx2>) -> bool {
_mm256_movemask_ps(_mm256_castsi256_ps(a.into())) as u32 != 0
}
);
kernel(self, a)
}
#[inline(always)]
fn all_true_mask32x8(self, a: mask32x8<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask32x8<Avx2>) -> bool {
_mm256_movemask_ps(_mm256_castsi256_ps(a.into())) as u32 == 0b11111111
}
);
kernel(self, a)
}
#[inline(always)]
fn any_false_mask32x8(self, a: mask32x8<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask32x8<Avx2>) -> bool {
_mm256_movemask_ps(_mm256_castsi256_ps(a.into())) as u32 != 0b11111111
}
);
kernel(self, a)
}
#[inline(always)]
fn all_false_mask32x8(self, a: mask32x8<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask32x8<Avx2>) -> bool {
_mm256_movemask_ps(_mm256_castsi256_ps(a.into())) as u32 == 0
}
);
kernel(self, a)
}
#[inline(always)]
fn combine_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x16<Self> {
mask32x16 {
val: crate::support::Aligned512([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn split_mask32x8(self, a: mask32x8<Self>) -> (mask32x4<Self>, mask32x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask32x8<Avx2>) -> (mask32x4<Avx2>, mask32x4<Avx2>) {
(
_mm256_extracti128_si256::<0>(a.into()).simd_into(token),
_mm256_extracti128_si256::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_f64x4(self, val: f64) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: f64) -> f64x4<Avx2> {
_mm256_set1_pd(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_f64x4<const SHIFT: usize>(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = cross_block_alignr_256x1(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn slide_within_blocks_f64x4<const SHIFT: usize>(
self,
a: f64x4<Self>,
b: f64x4<Self>,
) -> f64x4<Self> {
if SHIFT >= 2usize {
return b;
}
let result = dyn_alignr_256(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn abs_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>) -> f64x4<Avx2> {
_mm256_andnot_pd(_mm256_set1_pd(-0.0), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>) -> f64x4<Avx2> {
_mm256_xor_pd(a.into(), _mm256_set1_pd(-0.0)).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn sqrt_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>) -> f64x4<Avx2> {
_mm256_sqrt_pd(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn approximate_recip_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
1.0 / a
}
#[inline(always)]
fn add_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
_mm256_add_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
_mm256_sub_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
_mm256_mul_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn div_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
_mm256_div_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn copysign_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
let mask = _mm256_set1_pd(-0.0);
_mm256_or_pd(
_mm256_and_pd(mask, b.into()),
_mm256_andnot_pd(mask, a.into()),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
_mm256_max_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
_mm256_min_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_precise_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
let intermediate = _mm256_max_pd(a.into(), b.into());
let b_is_nan = _mm256_cmp_pd::<3i32>(b.into(), b.into());
_mm256_blendv_pd(intermediate, a.into(), b_is_nan).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_precise_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
let intermediate = _mm256_min_pd(a.into(), b.into());
let b_is_nan = _mm256_cmp_pd::<3i32>(b.into(), b.into());
_mm256_blendv_pd(intermediate, a.into(), b_is_nan).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> mask64x4<Avx2> {
_mm256_castpd_si256(_mm256_cmp_pd::<0i32>(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> mask64x4<Avx2> {
_mm256_castpd_si256(_mm256_cmp_pd::<17i32>(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> mask64x4<Avx2> {
_mm256_castpd_si256(_mm256_cmp_pd::<18i32>(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
let lo = _mm256_unpacklo_pd(a.into(), b.into());
let hi = _mm256_unpackhi_pd(a.into(), b.into());
_mm256_permute2f128_pd::<0b0010_0000>(lo, hi).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
let lo = _mm256_unpacklo_pd(a.into(), b.into());
let hi = _mm256_unpackhi_pd(a.into(), b.into());
_mm256_permute2f128_pd::<0b0011_0001>(lo, hi).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
let t1 = _mm256_permute4x64_pd::<0b11_01_10_00>(a.into());
let t2 = _mm256_permute4x64_pd::<0b11_01_10_00>(b.into());
_mm256_permute2f128_pd::<0b0010_0000>(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f64x4<Avx2> {
let t1 = _mm256_permute4x64_pd::<0b11_01_10_00>(a.into());
let t2 = _mm256_permute4x64_pd::<0b11_01_10_00>(b.into());
_mm256_permute2f128_pd::<0b0011_0001>(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> (f64x4<Self>, f64x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> (f64x4<Avx2>, f64x4<Avx2>) {
let lo = _mm256_unpacklo_pd(a.into(), b.into());
let hi = _mm256_unpackhi_pd(a.into(), b.into());
(
_mm256_permute2f128_pd::<0b0010_0000>(lo, hi).simd_into(token),
_mm256_permute2f128_pd::<0b0011_0001>(lo, hi).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> (f64x4<Self>, f64x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> (f64x4<Avx2>, f64x4<Avx2>) {
let t1 = _mm256_permute4x64_pd::<0b11_01_10_00>(a.into());
let t2 = _mm256_permute4x64_pd::<0b11_01_10_00>(b.into());
(
_mm256_permute2f128_pd::<0b0010_0000>(t1, t2).simd_into(token),
_mm256_permute2f128_pd::<0b0011_0001>(t1, t2).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_add_f64x4(self, a: f64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>, c: f64x4<Avx2>) -> f64x4<Avx2> {
_mm256_fmadd_pd(a.into(), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn mul_sub_f64x4(self, a: f64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>, c: f64x4<Avx2>) -> f64x4<Avx2> {
_mm256_fmsub_pd(a.into(), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn floor_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>) -> f64x4<Avx2> {
_mm256_round_pd::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn ceil_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>) -> f64x4<Avx2> {
_mm256_round_pd::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn round_ties_even_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>) -> f64x4<Avx2> {
_mm256_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn fract_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
a - self.trunc_f64x4(a)
}
#[inline(always)]
fn trunc_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>) -> f64x4<Avx2> {
_mm256_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn select_f64x4(self, a: mask64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask64x4<Avx2>,
b: f64x4<Avx2>,
c: f64x4<Avx2>,
) -> f64x4<Avx2> {
_mm256_blendv_pd(c.into(), b.into(), _mm256_castsi256_pd(a.into())).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x8<Self> {
f64x8 {
val: crate::support::Aligned512([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn split_f64x4(self, a: f64x4<Self>) -> (f64x2<Self>, f64x2<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>) -> (f64x2<Avx2>, f64x2<Avx2>) {
(
_mm256_extractf128_pd::<0>(a.into()).simd_into(token),
_mm256_extractf128_pd::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn narrow_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: f64x4<Avx2>, b: f64x4<Avx2>) -> f32x8<Avx2> {
let low = _mm256_cvtpd_ps(a.into());
let high = _mm256_cvtpd_ps(b.into());
_mm256_insertf128_ps::<1>(_mm256_castps128_ps256(low), high).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f32x8<Self> {
self.narrow_f64x4(a, b)
}
#[inline(always)]
fn relaxed_narrow_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f32x8<Self> {
self.narrow_f64x4(a, b)
}
#[inline(always)]
fn cvt_u64_f64x4(self, a: f64x4<Self>) -> u64x4<Self> {
[
a[0usize] as u64,
a[1usize] as u64,
a[2usize] as u64,
a[3usize] as u64,
]
.simd_into(self)
}
#[inline(always)]
fn cvt_u64_precise_f64x4(self, a: f64x4<Self>) -> u64x4<Self> {
[
a[0usize] as u64,
a[1usize] as u64,
a[2usize] as u64,
a[3usize] as u64,
]
.simd_into(self)
}
#[inline(always)]
fn cvt_i64_f64x4(self, a: f64x4<Self>) -> i64x4<Self> {
[
a[0usize] as i64,
a[1usize] as i64,
a[2usize] as i64,
a[3usize] as i64,
]
.simd_into(self)
}
#[inline(always)]
fn cvt_i64_precise_f64x4(self, a: f64x4<Self>) -> i64x4<Self> {
[
a[0usize] as i64,
a[1usize] as i64,
a[2usize] as i64,
a[3usize] as i64,
]
.simd_into(self)
}
#[inline(always)]
fn splat_i64x4(self, val: i64) -> i64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: i64) -> i64x4<Avx2> {
_mm256_set1_epi64x(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_i64x4<const SHIFT: usize>(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = cross_block_alignr_256x1(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn slide_within_blocks_i64x4<const SHIFT: usize>(
self,
a: i64x4<Self>,
b: i64x4<Self>,
) -> i64x4<Self> {
if SHIFT >= 2usize {
return b;
}
let result = dyn_alignr_256(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn add_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
_mm256_add_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
_mm256_sub_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
[
i64::wrapping_mul(a[0usize], b[0usize]),
i64::wrapping_mul(a[1usize], b[1usize]),
i64::wrapping_mul(a[2usize], b[2usize]),
i64::wrapping_mul(a[3usize], b[3usize]),
]
.simd_into(self)
}
#[inline(always)]
fn and_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
_mm256_and_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
_mm256_or_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
_mm256_xor_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i64x4(self, a: i64x4<Self>) -> i64x4<Self> {
a ^ !0
}
#[inline(always)]
fn shl_i64x4(self, a: i64x4<Self>, shift: u32) -> i64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x4<Avx2>, shift: u32) -> i64x4<Avx2> {
_mm256_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
_mm256_sllv_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_i64x4(self, a: i64x4<Self>, shift: u32) -> i64x4<Self> {
[
i64::wrapping_shr(a[0usize], shift),
i64::wrapping_shr(a[1usize], shift),
i64::wrapping_shr(a[2usize], shift),
i64::wrapping_shr(a[3usize], shift),
]
.simd_into(self)
}
#[inline(always)]
fn shrv_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
let value = a.into();
let counts = b.into();
let bias = _mm256_set1_epi64x(i64::MIN);
let shifted_bias = _mm256_srlv_epi64(bias, counts);
let shifted = _mm256_srlv_epi64(value, counts);
_mm256_sub_epi64(_mm256_xor_si256(shifted, shifted_bias), shifted_bias)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
[
i64::max(a[0usize], b[0usize]),
i64::max(a[1usize], b[1usize]),
i64::max(a[2usize], b[2usize]),
i64::max(a[3usize], b[3usize]),
]
.simd_into(self)
}
#[inline(always)]
fn min_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
[
i64::min(a[0usize], b[0usize]),
i64::min(a[1usize], b[1usize]),
i64::min(a[2usize], b[2usize]),
i64::min(a[3usize], b[3usize]),
]
.simd_into(self)
}
#[inline(always)]
fn simd_eq_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> mask64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> mask64x4<Avx2> {
_mm256_cmpeq_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> mask64x4<Self> {
[
-(i64::lt(&a[0usize], &b[0usize]) as i64),
-(i64::lt(&a[1usize], &b[1usize]) as i64),
-(i64::lt(&a[2usize], &b[2usize]) as i64),
-(i64::lt(&a[3usize], &b[3usize]) as i64),
]
.simd_into(self)
}
#[inline(always)]
fn simd_le_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> mask64x4<Self> {
[
-(i64::le(&a[0usize], &b[0usize]) as i64),
-(i64::le(&a[1usize], &b[1usize]) as i64),
-(i64::le(&a[2usize], &b[2usize]) as i64),
-(i64::le(&a[3usize], &b[3usize]) as i64),
]
.simd_into(self)
}
#[inline(always)]
fn zip_low_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
let lo = _mm256_unpacklo_epi64(a.into(), b.into());
let hi = _mm256_unpackhi_epi64(a.into(), b.into());
_mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
let lo = _mm256_unpacklo_epi64(a.into(), b.into());
let hi = _mm256_unpackhi_epi64(a.into(), b.into());
_mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(a.into());
let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(b.into());
_mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i64x4<Avx2> {
let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(a.into());
let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(b.into());
_mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> (i64x4<Self>, i64x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> (i64x4<Avx2>, i64x4<Avx2>) {
let lo = _mm256_unpacklo_epi64(a.into(), b.into());
let hi = _mm256_unpackhi_epi64(a.into(), b.into());
(
_mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token),
_mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> (i64x4<Self>, i64x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> (i64x4<Avx2>, i64x4<Avx2>) {
let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(a.into());
let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(b.into());
(
_mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token),
_mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_i64x4(self, a: mask64x4<Self>, b: i64x4<Self>, c: i64x4<Self>) -> i64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask64x4<Avx2>,
b: i64x4<Avx2>,
c: i64x4<Avx2>,
) -> i64x4<Avx2> {
_mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x8<Self> {
i64x8 {
val: crate::support::Aligned512([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn split_i64x4(self, a: i64x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x4<Avx2>) -> (i64x2<Avx2>, i64x2<Avx2>) {
(
_mm256_extracti128_si256::<0>(a.into()).simd_into(token),
_mm256_extracti128_si256::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_i64x4(self, a: i64x4<Self>) -> i64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x4<Avx2>) -> i64x4<Avx2> {
_mm256_sub_epi64(_mm256_setzero_si256(), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn narrow_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i32x8<Avx2> {
_mm256_permute4x64_epi64::<0xd8>(_mm256_castps_si256(_mm256_shuffle_ps::<0x88>(
_mm256_castsi256_ps(a.into()),
_mm256_castsi256_ps(b.into()),
)))
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: i64x4<Avx2>, b: i64x4<Avx2>) -> i32x8<Avx2> {
let a = a.into();
let b = b.into();
let low = _mm256_permute4x64_epi64::<0xd8>(_mm256_castps_si256(
_mm256_shuffle_ps::<0x88>(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b)),
));
let high = _mm256_permute4x64_epi64::<0xd8>(_mm256_castps_si256(
_mm256_shuffle_ps::<0xdd>(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b)),
));
let low_sign = _mm256_srai_epi32::<31>(low);
let fits = _mm256_cmpeq_epi32(high, low_sign);
let high_sign = _mm256_srai_epi32::<31>(high);
let bound = _mm256_xor_si256(high_sign, _mm256_set1_epi32(i32::MAX));
_mm256_blendv_epi8(bound, low, fits).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn relaxed_narrow_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i32x8<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= i32::MIN as i64 && value <= i32::MAX as i64 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.narrow_i64x4(a, b)
}
#[inline(always)]
fn cvt_f64_i64x4(self, a: i64x4<Self>) -> f64x4<Self> {
[
a[0usize] as f64,
a[1usize] as f64,
a[2usize] as f64,
a[3usize] as f64,
]
.simd_into(self)
}
#[inline(always)]
fn splat_u64x4(self, val: u64) -> u64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: u64) -> u64x4<Avx2> {
_mm256_set1_epi64x(val.cast_signed()).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_u64x4<const SHIFT: usize>(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = cross_block_alignr_256x1(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn slide_within_blocks_u64x4<const SHIFT: usize>(
self,
a: u64x4<Self>,
b: u64x4<Self>,
) -> u64x4<Self> {
if SHIFT >= 2usize {
return b;
}
let result = dyn_alignr_256(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn add_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
_mm256_add_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
_mm256_sub_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
[
u64::wrapping_mul(a[0usize], b[0usize]),
u64::wrapping_mul(a[1usize], b[1usize]),
u64::wrapping_mul(a[2usize], b[2usize]),
u64::wrapping_mul(a[3usize], b[3usize]),
]
.simd_into(self)
}
#[inline(always)]
fn and_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
_mm256_and_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
_mm256_or_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
_mm256_xor_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u64x4(self, a: u64x4<Self>) -> u64x4<Self> {
a ^ !0
}
#[inline(always)]
fn shl_u64x4(self, a: u64x4<Self>, shift: u32) -> u64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x4<Avx2>, shift: u32) -> u64x4<Avx2> {
_mm256_sll_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
_mm256_sllv_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_u64x4(self, a: u64x4<Self>, shift: u32) -> u64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x4<Avx2>, shift: u32) -> u64x4<Avx2> {
_mm256_srl_epi64(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
_mm256_srlv_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
[
u64::max(a[0usize], b[0usize]),
u64::max(a[1usize], b[1usize]),
u64::max(a[2usize], b[2usize]),
u64::max(a[3usize], b[3usize]),
]
.simd_into(self)
}
#[inline(always)]
fn min_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
[
u64::min(a[0usize], b[0usize]),
u64::min(a[1usize], b[1usize]),
u64::min(a[2usize], b[2usize]),
u64::min(a[3usize], b[3usize]),
]
.simd_into(self)
}
#[inline(always)]
fn simd_eq_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> mask64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> mask64x4<Avx2> {
_mm256_cmpeq_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> mask64x4<Self> {
[
-(u64::lt(&a[0usize], &b[0usize]) as i64),
-(u64::lt(&a[1usize], &b[1usize]) as i64),
-(u64::lt(&a[2usize], &b[2usize]) as i64),
-(u64::lt(&a[3usize], &b[3usize]) as i64),
]
.simd_into(self)
}
#[inline(always)]
fn simd_le_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> mask64x4<Self> {
[
-(u64::le(&a[0usize], &b[0usize]) as i64),
-(u64::le(&a[1usize], &b[1usize]) as i64),
-(u64::le(&a[2usize], &b[2usize]) as i64),
-(u64::le(&a[3usize], &b[3usize]) as i64),
]
.simd_into(self)
}
#[inline(always)]
fn zip_low_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
let lo = _mm256_unpacklo_epi64(a.into(), b.into());
let hi = _mm256_unpackhi_epi64(a.into(), b.into());
_mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
let lo = _mm256_unpacklo_epi64(a.into(), b.into());
let hi = _mm256_unpackhi_epi64(a.into(), b.into());
_mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(a.into());
let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(b.into());
_mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u64x4<Avx2> {
let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(a.into());
let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(b.into());
_mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> (u64x4<Self>, u64x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> (u64x4<Avx2>, u64x4<Avx2>) {
let lo = _mm256_unpacklo_epi64(a.into(), b.into());
let hi = _mm256_unpackhi_epi64(a.into(), b.into());
(
_mm256_permute2x128_si256::<0b0010_0000>(lo, hi).simd_into(token),
_mm256_permute2x128_si256::<0b0011_0001>(lo, hi).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> (u64x4<Self>, u64x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> (u64x4<Avx2>, u64x4<Avx2>) {
let t1 = _mm256_permute4x64_epi64::<0b11_01_10_00>(a.into());
let t2 = _mm256_permute4x64_epi64::<0b11_01_10_00>(b.into());
(
_mm256_permute2x128_si256::<0b0010_0000>(t1, t2).simd_into(token),
_mm256_permute2x128_si256::<0b0011_0001>(t1, t2).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_u64x4(self, a: mask64x4<Self>, b: u64x4<Self>, c: u64x4<Self>) -> u64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask64x4<Avx2>,
b: u64x4<Avx2>,
c: u64x4<Avx2>,
) -> u64x4<Avx2> {
_mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x8<Self> {
u64x8 {
val: crate::support::Aligned512([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn split_u64x4(self, a: u64x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x4<Avx2>) -> (u64x2<Avx2>, u64x2<Avx2>) {
(
_mm256_extracti128_si256::<0>(a.into()).simd_into(token),
_mm256_extracti128_si256::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn narrow_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u32x8<Avx2> {
_mm256_permute4x64_epi64::<0xd8>(_mm256_castps_si256(_mm256_shuffle_ps::<0x88>(
_mm256_castsi256_ps(a.into()),
_mm256_castsi256_ps(b.into()),
)))
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u64x4<Avx2>, b: u64x4<Avx2>) -> u32x8<Avx2> {
let zero = _mm256_setzero_si256();
let ones = _mm256_cmpeq_epi64(zero, zero);
let a = a.into();
let b = b.into();
let a_fits = _mm256_cmpeq_epi64(_mm256_srli_epi64::<32>(a), zero);
let b_fits = _mm256_cmpeq_epi64(_mm256_srli_epi64::<32>(b), zero);
let a = _mm256_or_si256(a, _mm256_xor_si256(a_fits, ones));
let b = _mm256_or_si256(b, _mm256_xor_si256(b_fits, ones));
_mm256_permute4x64_epi64::<0xd8>(_mm256_castps_si256(_mm256_shuffle_ps::<0x88>(
_mm256_castsi256_ps(a),
_mm256_castsi256_ps(b),
)))
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn relaxed_narrow_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u32x8<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= u32::MIN as u64 && value <= u32::MAX as u64 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.narrow_u64x4(a, b)
}
#[inline(always)]
fn cvt_f64_u64x4(self, a: u64x4<Self>) -> f64x4<Self> {
[
a[0usize] as f64,
a[1usize] as f64,
a[2usize] as f64,
a[3usize] as f64,
]
.simd_into(self)
}
#[inline(always)]
fn splat_mask64x4(self, val: bool) -> mask64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, val: bool) -> mask64x4<Avx2> {
let val: i64 = if val { !0 } else { 0 };
_mm256_set1_epi64x(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn from_bitmask_mask64x4(self, bits: u64) -> mask64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, bits: u64) -> mask64x4<Avx2> {
{
let bit_lanes = _mm256_set1_epi64x(bits.cast_signed());
let bit_mask = _mm256_set_epi64x(8, 4, 2, 1);
_mm256_cmpeq_epi64(_mm256_and_si256(bit_lanes, bit_mask), bit_mask)
}
.simd_into(token)
}
);
kernel(self, bits)
}
#[inline(always)]
fn to_bitmask_mask64x4(self, a: mask64x4<Self>) -> u64 {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask64x4<Avx2>) -> u64 {
_mm256_movemask_pd(_mm256_castsi256_pd(a.into())) as u32 as u64
}
);
kernel(self, a)
}
#[inline(always)]
fn set_mask64x4(self, a: &mut mask64x4<Self>, index: usize, value: bool) -> () {
assert!(
index < 4usize,
"mask lane index {index} is out of bounds for {} lanes",
4usize
);
let mut lanes: [i64; 4usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn and_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask64x4<Avx2>, b: mask64x4<Avx2>) -> mask64x4<Avx2> {
_mm256_and_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask64x4<Avx2>, b: mask64x4<Avx2>) -> mask64x4<Avx2> {
_mm256_or_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask64x4<Avx2>, b: mask64x4<Avx2>) -> mask64x4<Avx2> {
_mm256_xor_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_mask64x4(self, a: mask64x4<Self>) -> mask64x4<Self> {
self.xor_mask64x4(a, self.splat_mask64x4(true))
}
#[inline(always)]
fn select_mask64x4(
self,
a: mask64x4<Self>,
b: mask64x4<Self>,
c: mask64x4<Self>,
) -> mask64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx2,
a: mask64x4<Avx2>,
b: mask64x4<Avx2>,
c: mask64x4<Avx2>,
) -> mask64x4<Avx2> {
_mm256_blendv_epi8(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn simd_eq_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask64x4<Avx2>, b: mask64x4<Avx2>) -> mask64x4<Avx2> {
_mm256_cmpeq_epi64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn any_true_mask64x4(self, a: mask64x4<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask64x4<Avx2>) -> bool {
_mm256_movemask_pd(_mm256_castsi256_pd(a.into())) as u32 != 0
}
);
kernel(self, a)
}
#[inline(always)]
fn all_true_mask64x4(self, a: mask64x4<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask64x4<Avx2>) -> bool {
_mm256_movemask_pd(_mm256_castsi256_pd(a.into())) as u32 == 0b1111
}
);
kernel(self, a)
}
#[inline(always)]
fn any_false_mask64x4(self, a: mask64x4<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask64x4<Avx2>) -> bool {
_mm256_movemask_pd(_mm256_castsi256_pd(a.into())) as u32 != 0b1111
}
);
kernel(self, a)
}
#[inline(always)]
fn all_false_mask64x4(self, a: mask64x4<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask64x4<Avx2>) -> bool {
_mm256_movemask_pd(_mm256_castsi256_pd(a.into())) as u32 == 0
}
);
kernel(self, a)
}
#[inline(always)]
fn combine_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x8<Self> {
mask64x8 {
val: crate::support::Aligned512([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn split_mask64x4(self, a: mask64x4<Self>) -> (mask64x2<Self>, mask64x2<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask64x4<Avx2>) -> (mask64x2<Avx2>, mask64x2<Avx2>) {
(
_mm256_extracti128_si256::<0>(a.into()).simd_into(token),
_mm256_extracti128_si256::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn slide_f32x16<const SHIFT: usize>(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = cross_block_alignr_256x2(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_f32x16(self, a: f32x16<Self>) -> (f32x8<Self>, f32x8<Self>) {
(
f32x8 {
val: crate::support::Aligned256(a.val.0[0]),
simd: self,
},
f32x8 {
val: crate::support::Aligned256(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn slide_i8x64<const SHIFT: usize>(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
if SHIFT >= 64usize {
return b;
}
let result = cross_block_alignr_256x2(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT,
);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_i8x64(self, a: i8x64<Self>) -> (i8x32<Self>, i8x32<Self>) {
(
i8x32 {
val: crate::support::Aligned256(a.val.0[0]),
simd: self,
},
i8x32 {
val: crate::support::Aligned256(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn slide_u8x64<const SHIFT: usize>(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
if SHIFT >= 64usize {
return b;
}
let result = cross_block_alignr_256x2(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT,
);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_u8x64(self, a: u8x64<Self>, indices: u8x64<Self>) -> u8x64<Self> {
self.swizzle_dyn_precise_u8x64(a, indices)
}
#[inline(always)]
fn swizzle_dyn_precise_u8x64(self, a: u8x64<Self>, indices: u8x64<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: u8x64<Avx2>, indices: u8x64<Avx2>) -> u8x64<Avx2> {
let bytes = Bytes::to_bytes(a);
let (table_low, table_high) = token.split_u8x64(bytes);
let (indices_low, indices_high) = token.split_u8x64(indices);
let high_table_offset = token.splat_u8x32(32);
let output_low_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_low);
let output_low_from_high = token.swizzle_dyn_precise_u8x32(
table_high,
token.sub_u8x32(indices_low, high_table_offset),
);
let output_low = token.or_u8x32(output_low_from_low, output_low_from_high);
let output_high_from_low = token.swizzle_dyn_precise_u8x32(table_low, indices_high);
let output_high_from_high = token.swizzle_dyn_precise_u8x32(
table_high,
token.sub_u8x32(indices_high, high_table_offset),
);
let output_high = token.or_u8x32(output_high_from_low, output_high_from_high);
let result_bytes = token.combine_u8x32(output_low, output_high);
Bytes::from_bytes(result_bytes)
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn split_u8x64(self, a: u8x64<Self>) -> (u8x32<Self>, u8x32<Self>) {
(
u8x32 {
val: crate::support::Aligned256(a.val.0[0]),
simd: self,
},
u8x32 {
val: crate::support::Aligned256(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn from_bitmask_mask8x64(self, bits: u64) -> mask8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, bits: u64) -> mask8x64<Avx2> {
{
let bit_bytes = _mm256_set1_epi64x(bits.cast_signed());
let bit_mask = _mm256_setr_epi8(
1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, 8, 16,
32, 64, -128, 1, 2, 4, 8, 16, 32, 64, -128,
);
mask8x64 {
val: crate::support::Aligned512([
{
let bit_bytes = _mm256_shuffle_epi8(
bit_bytes,
_mm256_setr_epi8(
0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2,
2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3,
),
);
_mm256_cmpeq_epi8(_mm256_and_si256(bit_bytes, bit_mask), bit_mask)
},
{
let bit_bytes = _mm256_shuffle_epi8(
bit_bytes,
_mm256_setr_epi8(
4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6,
6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7,
),
);
_mm256_cmpeq_epi8(_mm256_and_si256(bit_bytes, bit_mask), bit_mask)
},
]),
simd: token,
}
}
}
);
kernel(self, bits)
}
#[inline(always)]
fn set_mask8x64(self, a: &mut mask8x64<Self>, index: usize, value: bool) -> () {
assert!(
index < 64usize,
"mask lane index {index} is out of bounds for {} lanes",
64usize
);
let mut lanes: [i8; 64usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn split_mask8x64(self, a: mask8x64<Self>) -> (mask8x32<Self>, mask8x32<Self>) {
(
mask8x32 {
val: crate::support::Aligned256(a.val.0[0]),
simd: self,
},
mask8x32 {
val: crate::support::Aligned256(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn slide_i16x32<const SHIFT: usize>(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
if SHIFT >= 32usize {
return b;
}
let result = cross_block_alignr_256x2(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 2usize,
);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_i16x32(self, a: i16x32<Self>) -> (i16x16<Self>, i16x16<Self>) {
(
i16x16 {
val: crate::support::Aligned256(a.val.0[0]),
simd: self,
},
i16x16 {
val: crate::support::Aligned256(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn slide_u16x32<const SHIFT: usize>(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
if SHIFT >= 32usize {
return b;
}
let result = cross_block_alignr_256x2(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 2usize,
);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_u16x32(self, a: u16x32<Self>) -> (u16x16<Self>, u16x16<Self>) {
(
u16x16 {
val: crate::support::Aligned256(a.val.0[0]),
simd: self,
},
u16x16 {
val: crate::support::Aligned256(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn to_bitmask_mask16x32(self, a: mask16x32<Self>) -> u64 {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, a: mask16x32<Avx2>) -> u64 {
{
let lo = _mm256_movemask_epi8(a.val.0[0]) as u32;
let hi = _mm256_movemask_epi8(a.val.0[1]) as u32;
let lo = _pext_u32(lo, 0x5555_5555u32) as u64;
let hi = _pext_u32(hi, 0x5555_5555u32) as u64;
lo | (hi << 16usize)
}
}
);
kernel(self, a)
}
#[inline(always)]
fn set_mask16x32(self, a: &mut mask16x32<Self>, index: usize, value: bool) -> () {
assert!(
index < 32usize,
"mask lane index {index} is out of bounds for {} lanes",
32usize
);
let mut lanes: [i16; 32usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn split_mask16x32(self, a: mask16x32<Self>) -> (mask16x16<Self>, mask16x16<Self>) {
(
mask16x16 {
val: crate::support::Aligned256(a.val.0[0]),
simd: self,
},
mask16x16 {
val: crate::support::Aligned256(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn slide_i32x16<const SHIFT: usize>(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = cross_block_alignr_256x2(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_i32x16(self, a: i32x16<Self>) -> (i32x8<Self>, i32x8<Self>) {
(
i32x8 {
val: crate::support::Aligned256(a.val.0[0]),
simd: self,
},
i32x8 {
val: crate::support::Aligned256(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn slide_u32x16<const SHIFT: usize>(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = cross_block_alignr_256x2(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_u32x16(self, a: u32x16<Self>) -> (u32x8<Self>, u32x8<Self>) {
(
u32x8 {
val: crate::support::Aligned256(a.val.0[0]),
simd: self,
},
u32x8 {
val: crate::support::Aligned256(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn from_bitmask_mask32x16(self, bits: u64) -> mask32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, bits: u64) -> mask32x16<Avx2> {
{
let bit_lanes = _mm256_set1_epi32(bits as i32);
mask32x16 {
val: crate::support::Aligned512([
{
let bit_mask = _mm256_setr_epi32(1, 2, 4, 8, 16, 32, 64, 128);
_mm256_cmpeq_epi32(_mm256_and_si256(bit_lanes, bit_mask), bit_mask)
},
{
let bit_mask = _mm256_setr_epi32(
256, 512, 1024, 2048, 4096, 8192, 16384, 32768,
);
_mm256_cmpeq_epi32(_mm256_and_si256(bit_lanes, bit_mask), bit_mask)
},
]),
simd: token,
}
}
}
);
kernel(self, bits)
}
#[inline(always)]
fn set_mask32x16(self, a: &mut mask32x16<Self>, index: usize, value: bool) -> () {
assert!(
index < 16usize,
"mask lane index {index} is out of bounds for {} lanes",
16usize
);
let mut lanes: [i32; 16usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn split_mask32x16(self, a: mask32x16<Self>) -> (mask32x8<Self>, mask32x8<Self>) {
(
mask32x8 {
val: crate::support::Aligned256(a.val.0[0]),
simd: self,
},
mask32x8 {
val: crate::support::Aligned256(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn slide_f64x8<const SHIFT: usize>(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = cross_block_alignr_256x2(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_f64x8(self, a: f64x8<Self>) -> (f64x4<Self>, f64x4<Self>) {
(
f64x4 {
val: crate::support::Aligned256(a.val.0[0]),
simd: self,
},
f64x4 {
val: crate::support::Aligned256(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn slide_i64x8<const SHIFT: usize>(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = cross_block_alignr_256x2(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_i64x8(self, a: i64x8<Self>) -> (i64x4<Self>, i64x4<Self>) {
(
i64x4 {
val: crate::support::Aligned256(a.val.0[0]),
simd: self,
},
i64x4 {
val: crate::support::Aligned256(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn slide_u64x8<const SHIFT: usize>(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = cross_block_alignr_256x2(
self,
Bytes::to_bytes(b).val.0,
Bytes::to_bytes(a).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_u64x8(self, a: u64x8<Self>) -> (u64x4<Self>, u64x4<Self>) {
(
u64x4 {
val: crate::support::Aligned256(a.val.0[0]),
simd: self,
},
u64x4 {
val: crate::support::Aligned256(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn from_bitmask_mask64x8(self, bits: u64) -> mask64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx2, bits: u64) -> mask64x8<Avx2> {
{
let bit_lanes = _mm256_set1_epi64x(bits.cast_signed());
mask64x8 {
val: crate::support::Aligned512([
{
let bit_mask = _mm256_set_epi64x(8, 4, 2, 1);
_mm256_cmpeq_epi64(_mm256_and_si256(bit_lanes, bit_mask), bit_mask)
},
{
let bit_mask = _mm256_set_epi64x(128, 64, 32, 16);
_mm256_cmpeq_epi64(_mm256_and_si256(bit_lanes, bit_mask), bit_mask)
},
]),
simd: token,
}
}
}
);
kernel(self, bits)
}
#[inline(always)]
fn set_mask64x8(self, a: &mut mask64x8<Self>, index: usize, value: bool) -> () {
assert!(
index < 8usize,
"mask lane index {index} is out of bounds for {} lanes",
8usize
);
let mut lanes: [i64; 8usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn split_mask64x8(self, a: mask64x8<Self>) -> (mask64x4<Self>, mask64x4<Self>) {
(
mask64x4 {
val: crate::support::Aligned256(a.val.0[0]),
simd: self,
},
mask64x4 {
val: crate::support::Aligned256(a.val.0[1]),
simd: self,
},
)
}
}
impl<S: Simd> SimdFrom<__m256, S> for f32x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m256) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<f32x8<S>> for __m256 {
#[inline(always)]
fn from(value: f32x8<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<__m256i, S> for i8x32<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m256i) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i8x32<S>> for __m256i {
#[inline(always)]
fn from(value: i8x32<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<__m256i, S> for u8x32<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m256i) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u8x32<S>> for __m256i {
#[inline(always)]
fn from(value: u8x32<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<__m256i, S> for mask8x32<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m256i) -> Self {
let lanes: [i8; 32usize] = crate::transmute::checked_transmute_copy(&arch);
lanes.simd_into(simd)
}
}
impl<S: Simd> From<mask8x32<S>> for __m256i {
#[inline(always)]
fn from(value: mask8x32<S>) -> Self {
let lanes: [i8; 32usize] = value.into();
crate::transmute::checked_transmute_copy(&lanes)
}
}
impl<S: Simd> SimdFrom<__m256i, S> for i16x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m256i) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i16x16<S>> for __m256i {
#[inline(always)]
fn from(value: i16x16<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<__m256i, S> for u16x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m256i) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u16x16<S>> for __m256i {
#[inline(always)]
fn from(value: u16x16<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<__m256i, S> for mask16x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m256i) -> Self {
let lanes: [i16; 16usize] = crate::transmute::checked_transmute_copy(&arch);
lanes.simd_into(simd)
}
}
impl<S: Simd> From<mask16x16<S>> for __m256i {
#[inline(always)]
fn from(value: mask16x16<S>) -> Self {
let lanes: [i16; 16usize] = value.into();
crate::transmute::checked_transmute_copy(&lanes)
}
}
impl<S: Simd> SimdFrom<__m256i, S> for i32x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m256i) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i32x8<S>> for __m256i {
#[inline(always)]
fn from(value: i32x8<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<__m256i, S> for u32x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m256i) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u32x8<S>> for __m256i {
#[inline(always)]
fn from(value: u32x8<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<__m256i, S> for mask32x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m256i) -> Self {
let lanes: [i32; 8usize] = crate::transmute::checked_transmute_copy(&arch);
lanes.simd_into(simd)
}
}
impl<S: Simd> From<mask32x8<S>> for __m256i {
#[inline(always)]
fn from(value: mask32x8<S>) -> Self {
let lanes: [i32; 8usize] = value.into();
crate::transmute::checked_transmute_copy(&lanes)
}
}
impl<S: Simd> SimdFrom<__m256d, S> for f64x4<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m256d) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<f64x4<S>> for __m256d {
#[inline(always)]
fn from(value: f64x4<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<__m256i, S> for i64x4<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m256i) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i64x4<S>> for __m256i {
#[inline(always)]
fn from(value: i64x4<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<__m256i, S> for u64x4<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m256i) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u64x4<S>> for __m256i {
#[inline(always)]
fn from(value: u64x4<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<__m256i, S> for mask64x4<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m256i) -> Self {
let lanes: [i64; 4usize] = crate::transmute::checked_transmute_copy(&arch);
lanes.simd_into(simd)
}
}
impl<S: Simd> From<mask64x4<S>> for __m256i {
#[inline(always)]
fn from(value: mask64x4<S>) -> Self {
let lanes: [i64; 4usize] = value.into();
crate::transmute::checked_transmute_copy(&lanes)
}
}
crate::kernel!(
#[doc = r" This is a version of the `alignr` intrinsic that takes a non-const shift argument. The shift is still"]
#[doc = r" expected to be constant in practice, so the match statement will be optimized out. This exists because"]
#[doc = r" Rust doesn't currently let you do math on const generics."]
#[inline(always)]
fn dyn_alignr_128(token: Avx2, a: __m128i, b: __m128i, shift: usize) -> __m128i {
match shift {
0usize => _mm_alignr_epi8::<0i32>(a, b),
1usize => _mm_alignr_epi8::<1i32>(a, b),
2usize => _mm_alignr_epi8::<2i32>(a, b),
3usize => _mm_alignr_epi8::<3i32>(a, b),
4usize => _mm_alignr_epi8::<4i32>(a, b),
5usize => _mm_alignr_epi8::<5i32>(a, b),
6usize => _mm_alignr_epi8::<6i32>(a, b),
7usize => _mm_alignr_epi8::<7i32>(a, b),
8usize => _mm_alignr_epi8::<8i32>(a, b),
9usize => _mm_alignr_epi8::<9i32>(a, b),
10usize => _mm_alignr_epi8::<10i32>(a, b),
11usize => _mm_alignr_epi8::<11i32>(a, b),
12usize => _mm_alignr_epi8::<12i32>(a, b),
13usize => _mm_alignr_epi8::<13i32>(a, b),
14usize => _mm_alignr_epi8::<14i32>(a, b),
15usize => _mm_alignr_epi8::<15i32>(a, b),
_ => unreachable!(),
}
}
);
crate::kernel!(
#[doc = r" This is a version of the `alignr` intrinsic that takes a non-const shift argument. The shift is still"]
#[doc = r" expected to be constant in practice, so the match statement will be optimized out. This exists because"]
#[doc = r" Rust doesn't currently let you do math on const generics."]
#[inline(always)]
fn dyn_alignr_256(token: Avx2, a: __m256i, b: __m256i, shift: usize) -> __m256i {
match shift {
0usize => _mm256_alignr_epi8::<0i32>(a, b),
1usize => _mm256_alignr_epi8::<1i32>(a, b),
2usize => _mm256_alignr_epi8::<2i32>(a, b),
3usize => _mm256_alignr_epi8::<3i32>(a, b),
4usize => _mm256_alignr_epi8::<4i32>(a, b),
5usize => _mm256_alignr_epi8::<5i32>(a, b),
6usize => _mm256_alignr_epi8::<6i32>(a, b),
7usize => _mm256_alignr_epi8::<7i32>(a, b),
8usize => _mm256_alignr_epi8::<8i32>(a, b),
9usize => _mm256_alignr_epi8::<9i32>(a, b),
10usize => _mm256_alignr_epi8::<10i32>(a, b),
11usize => _mm256_alignr_epi8::<11i32>(a, b),
12usize => _mm256_alignr_epi8::<12i32>(a, b),
13usize => _mm256_alignr_epi8::<13i32>(a, b),
14usize => _mm256_alignr_epi8::<14i32>(a, b),
15usize => _mm256_alignr_epi8::<15i32>(a, b),
_ => unreachable!(),
}
}
);
crate::kernel!(
#[doc = r" Computes one output __m256i for `cross_block_alignr_*` operations."]
#[doc = r""]
#[doc = r" Given an array of registers, each containing two 128-bit blocks, extracts two adjacent blocks (`lo_idx` and"]
#[doc = r" `hi_idx` = `lo_idx + 1`) and performs `alignr` with `intra_shift`."]
#[inline(always)]
fn cross_block_alignr_one(
token: Avx2,
regs: &[__m256i],
block_idx: usize,
shift_bytes: usize,
) -> __m256i {
let lo_idx = block_idx + (shift_bytes / 16);
let intra_shift = shift_bytes % 16;
let lo_blocks = if lo_idx & 1 == 0 {
regs[lo_idx / 2]
} else {
_mm256_permute2x128_si256::<0x21>(regs[lo_idx / 2], regs[(lo_idx / 2) + 1])
};
let hi_idx = lo_idx + 1;
let hi_blocks = if hi_idx & 1 == 0 {
regs[hi_idx / 2]
} else {
_mm256_permute2x128_si256::<0x21>(regs[hi_idx / 2], regs[(hi_idx / 2) + 1])
};
dyn_alignr_256(token, hi_blocks, lo_blocks, intra_shift)
}
);
crate::kernel!(
#[doc = r" Concatenates `b` and `a` (each 2 x __m256i = 4 blocks) and extracts 4 blocks starting at byte offset"]
#[doc = r" `shift_bytes`. Extracts from [b : a] (b in low bytes, a in high bytes), matching alignr semantics."]
#[inline(always)]
fn cross_block_alignr_256x2(
token: Avx2,
a: [__m256i; 2],
b: [__m256i; 2],
shift_bytes: usize,
) -> [__m256i; 2] {
let regs = [b[0], b[1], a[0], a[1]];
[
cross_block_alignr_one(token, ®s, 0, shift_bytes),
cross_block_alignr_one(token, ®s, 2, shift_bytes),
]
}
);
crate::kernel!(
#[doc = r" Concatenates `b` and `a` (each 1 x __m256i = 2 blocks) and extracts 2 blocks starting at byte offset"]
#[doc = r" `shift_bytes`. Extracts from [b : a] (b in low bytes, a in high bytes), matching alignr semantics."]
#[inline(always)]
fn cross_block_alignr_256x1(
token: Avx2,
a: __m256i,
b: __m256i,
shift_bytes: usize,
) -> __m256i {
let regs = [b, a];
cross_block_alignr_one(token, ®s, 0, shift_bytes)
}
);