#![allow(
clippy::identity_op,
reason = "AVX-512 mask code is generated uniformly for all __mmask widths"
)]
#![allow(
clippy::useless_conversion,
reason = "AVX-512 mask code is generated uniformly for all __mmask widths"
)]
use crate::{Level, arch_types::ArchTypes, prelude::*, seal::Seal};
use crate::{
f32x4, f32x8, f32x16, f64x2, f64x4, f64x8, i8x16, i8x32, i8x64, i16x8, i16x16, i16x32, i32x4,
i32x8, i32x16, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16, mask16x32, mask32x4,
mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64, u16x8, u16x16, u16x32,
u32x4, u32x8, u32x16,
};
#[cfg(target_arch = "x86")]
use core::arch::x86::*;
#[cfg(target_arch = "x86_64")]
use core::arch::x86_64::*;
#[doc = "A token for AVX-512 intrinsics on `x86` and `x86_64`, representing an Ice Lake feature level."]
#[derive(Clone, Copy, Debug)]
pub struct Avx512 {
_private: (),
}
impl Avx512 {
#[doc = r" Create a SIMD token."]
#[doc = r""]
#[doc = r" # Safety"]
#[doc = r""]
#[doc = r" The Ice Lake AVX-512 CPU feature set must be available."]
#[inline]
pub const unsafe fn new_unchecked() -> Self {
Self { _private: () }
}
}
impl Seal for Avx512 {}
impl ArchTypes for Avx512 {
type f32x4 = crate::support::Aligned128<__m128>;
type i8x16 = crate::support::Aligned128<__m128i>;
type u8x16 = crate::support::Aligned128<__m128i>;
type mask8x16 = __mmask16;
type i16x8 = crate::support::Aligned128<__m128i>;
type u16x8 = crate::support::Aligned128<__m128i>;
type mask16x8 = __mmask8;
type i32x4 = crate::support::Aligned128<__m128i>;
type u32x4 = crate::support::Aligned128<__m128i>;
type mask32x4 = __mmask8;
type f64x2 = crate::support::Aligned128<__m128d>;
type mask64x2 = __mmask8;
type f32x8 = crate::support::Aligned256<__m256>;
type i8x32 = crate::support::Aligned256<__m256i>;
type u8x32 = crate::support::Aligned256<__m256i>;
type mask8x32 = __mmask32;
type i16x16 = crate::support::Aligned256<__m256i>;
type u16x16 = crate::support::Aligned256<__m256i>;
type mask16x16 = __mmask16;
type i32x8 = crate::support::Aligned256<__m256i>;
type u32x8 = crate::support::Aligned256<__m256i>;
type mask32x8 = __mmask8;
type f64x4 = crate::support::Aligned256<__m256d>;
type mask64x4 = __mmask8;
type f32x16 = crate::support::Aligned512<__m512>;
type i8x64 = crate::support::Aligned512<__m512i>;
type u8x64 = crate::support::Aligned512<__m512i>;
type mask8x64 = __mmask64;
type i16x32 = crate::support::Aligned512<__m512i>;
type u16x32 = crate::support::Aligned512<__m512i>;
type mask16x32 = __mmask32;
type i32x16 = crate::support::Aligned512<__m512i>;
type u32x16 = crate::support::Aligned512<__m512i>;
type mask32x16 = __mmask16;
type f64x8 = crate::support::Aligned512<__m512d>;
type mask64x8 = __mmask8;
}
impl Simd for Avx512 {
type f32s = f32x16<Self>;
type f64s = f64x8<Self>;
type u8s = u8x64<Self>;
type i8s = i8x64<Self>;
type u16s = u16x32<Self>;
type i16s = i16x32<Self>;
type u32s = u32x16<Self>;
type i32s = i32x16<Self>;
type mask8s = mask8x64<Self>;
type mask16s = mask16x32<Self>;
type mask32s = mask32x16<Self>;
type mask64s = mask64x8<Self>;
#[inline(always)]
fn level(self) -> Level {
Level::Avx512(self)
}
#[inline]
fn vectorize<F: FnOnce() -> R, R>(self, f: F) -> R {
#[target_feature(
enable = "adx,aes,avx512bitalg,avx512bw,avx512cd,avx512dq,avx512f,avx512ifma,avx512vbmi,avx512vbmi2,avx512vl,avx512vnni,avx512vpopcntdq,bmi1,bmi2,cmpxchg16b,fma,gfni,lzcnt,movbe,pclmulqdq,popcnt,rdrand,rdseed,sha,vaes,vpclmulqdq,xsave,xsavec,xsaveopt,xsaves"
)]
fn vectorize_avx512<F: FnOnce() -> R, R>(f: F) -> R {
f()
}
unsafe { vectorize_avx512(f) }
}
#[inline(always)]
fn splat_f32x4(self, val: f32) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: f32) -> f32x4<Avx512> {
_mm_set1_ps(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_f32x4(self, val: [f32; 4usize]) -> f32x4<Self> {
f32x4 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_f32x4(self, val: &[f32; 4usize]) -> f32x4<Self> {
f32x4 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_f32x4(self, a: f32x4<Self>) -> [f32; 4usize] {
crate::transmute::checked_transmute_copy::<__m128, [f32; 4usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_f32x4(self, a: &f32x4<Self>) -> &[f32; 4usize] {
crate::transmute::checked_cast_ref::<__m128, [f32; 4usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_f32x4(self, a: &mut f32x4<Self>) -> &mut [f32; 4usize] {
crate::transmute::checked_cast_mut::<__m128, [f32; 4usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_f32x4(self, a: f32x4<Self>, dest: &mut [f32; 4usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_f32x4(self, a: u8x16<Self>) -> f32x4<Self> {
f32x4 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_f32x4(self, a: f32x4<Self>) -> u8x16<Self> {
u8x16 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_f32x4<const SHIFT: usize>(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = dyn_alignr_128(
self,
self.cvt_to_bytes_f32x4(b).val.0,
self.cvt_to_bytes_f32x4(a).val.0,
SHIFT * 4usize,
);
self.cvt_from_bytes_f32x4(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn slide_within_blocks_f32x4<const SHIFT: usize>(
self,
a: f32x4<Self>,
b: f32x4<Self>,
) -> f32x4<Self> {
self.slide_f32x4::<SHIFT>(a, b)
}
#[inline(always)]
fn swizzle_dyn_within_blocks_f32x4(self, a: f32x4<Self>, indices: u8x16<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>, indices: u8x16<Avx512>) -> f32x4<Avx512> {
let bytes = token.cvt_to_bytes_f32x4(a).val.0;
let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into());
token.cvt_from_bytes_f32x4(u8x16 {
val: crate::support::Aligned128(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn abs_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32x4<Avx512> {
_mm_andnot_ps(_mm_set1_ps(-0.0), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32x4<Avx512> {
_mm_xor_ps(a.into(), _mm_set1_ps(-0.0)).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn sqrt_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32x4<Avx512> {
_mm_sqrt_ps(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn approximate_recip_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32x4<Avx512> {
_mm_rcp14_ps(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn add_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
_mm_add_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
_mm_sub_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
_mm_mul_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn div_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
_mm_div_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn copysign_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
let mask = _mm_set1_ps(-0.0);
_mm_or_ps(_mm_and_ps(mask, b.into()), _mm_andnot_ps(mask, a.into()))
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> mask32x4<Avx512> {
mask32x4 {
val: _mm_cmp_ps_mask::<0i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> mask32x4<Avx512> {
mask32x4 {
val: _mm_cmp_ps_mask::<17i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> mask32x4<Avx512> {
mask32x4 {
val: _mm_cmp_ps_mask::<18i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> mask32x4<Avx512> {
mask32x4 {
val: _mm_cmp_ps_mask::<29i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> mask32x4<Avx512> {
mask32x4 {
val: _mm_cmp_ps_mask::<30i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
_mm_unpacklo_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
_mm_unpackhi_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
_mm_shuffle_ps::<0b10_00_10_00>(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
_mm_shuffle_ps::<0b11_01_11_01>(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> (f32x4<Self>, f32x4<Self>) {
(self.zip_low_f32x4(a, b), self.zip_high_f32x4(a, b))
}
#[inline(always)]
fn deinterleave_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> (f32x4<Self>, f32x4<Self>) {
(self.unzip_low_f32x4(a, b), self.unzip_high_f32x4(a, b))
}
#[inline(always)]
fn max_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
_mm_max_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
_mm_min_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
_mm_range_ps::<5i32>(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x4<Avx512> {
_mm_range_ps::<4i32>(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_add_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f32x4<Avx512>,
b: f32x4<Avx512>,
c: f32x4<Avx512>,
) -> f32x4<Avx512> {
_mm_fmadd_ps(a.into(), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn mul_sub_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f32x4<Avx512>,
b: f32x4<Avx512>,
c: f32x4<Avx512>,
) -> f32x4<Avx512> {
_mm_fmsub_ps(a.into(), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn floor_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32x4<Avx512> {
_mm_round_ps::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn ceil_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32x4<Avx512> {
_mm_round_ps::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn round_ties_even_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32x4<Avx512> {
_mm_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn fract_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
a - self.trunc_f32x4(a)
}
#[inline(always)]
fn trunc_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>) -> f32x4<Avx512> {
_mm_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn select_f32x4(self, a: mask32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask32x4<Avx512>,
b: f32x4<Avx512>,
c: f32x4<Avx512>,
) -> f32x4<Avx512> {
_mm_mask_blend_ps(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>, b: f32x4<Avx512>) -> f32x8<Avx512> {
_mm256_setr_m128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn reinterpret_f64_f32x4(self, a: f32x4<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>) -> f64x2<Avx512> {
_mm_castps_pd(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_i32_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>) -> i32x4<Avx512> {
_mm_castps_si128(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u8_f32x4(self, a: f32x4<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>) -> u8x16<Avx512> {
_mm_castps_si128(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u32_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>) -> u32x4<Avx512> {
_mm_castps_si128(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_u32_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>) -> u32x4<Avx512> {
_mm_cvttps_epu32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_u32_precise_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>) -> u32x4<Avx512> {
let a = _mm_max_ps(a.into(), _mm_setzero_ps());
let mut converted = _mm_cvttps_epu32(a);
let exceeds_unsigned_range = _mm_cmp_ps_mask::<17i32>(_mm_set1_ps(4294967040.0), a);
converted = _mm_mask_blend_epi32(
exceeds_unsigned_range,
converted,
_mm_set1_epi32(u32::MAX.cast_signed()),
);
converted.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_i32_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>) -> i32x4<Avx512> {
_mm_cvttps_epi32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_i32_precise_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x4<Avx512>) -> i32x4<Avx512> {
let a = a.into();
let in_range = _mm_cmp_ps_mask::<17i32>(a, _mm_set1_ps(2147483648.0));
let mut converted = _mm_mask_cvttps_epi32(_mm_set1_epi32(i32::MAX), in_range, a);
let is_not_nan = _mm_cmp_ps_mask::<7i32>(a, a);
converted = _mm_mask_blend_epi32(is_not_nan, _mm_setzero_si128(), converted);
converted.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_i8x16(self, val: i8) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: i8) -> i8x16<Avx512> {
_mm_set1_epi8(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_i8x16(self, val: [i8; 16usize]) -> i8x16<Self> {
i8x16 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_i8x16(self, val: &[i8; 16usize]) -> i8x16<Self> {
i8x16 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_i8x16(self, a: i8x16<Self>) -> [i8; 16usize] {
crate::transmute::checked_transmute_copy::<__m128i, [i8; 16usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_i8x16(self, a: &i8x16<Self>) -> &[i8; 16usize] {
crate::transmute::checked_cast_ref::<__m128i, [i8; 16usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_i8x16(self, a: &mut i8x16<Self>) -> &mut [i8; 16usize] {
crate::transmute::checked_cast_mut::<__m128i, [i8; 16usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_i8x16(self, a: i8x16<Self>, dest: &mut [i8; 16usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_i8x16(self, a: u8x16<Self>) -> i8x16<Self> {
i8x16 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_i8x16(self, a: i8x16<Self>) -> u8x16<Self> {
u8x16 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_i8x16<const SHIFT: usize>(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = dyn_alignr_128(
self,
self.cvt_to_bytes_i8x16(b).val.0,
self.cvt_to_bytes_i8x16(a).val.0,
SHIFT,
);
self.cvt_from_bytes_i8x16(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn slide_within_blocks_i8x16<const SHIFT: usize>(
self,
a: i8x16<Self>,
b: i8x16<Self>,
) -> i8x16<Self> {
self.slide_i8x16::<SHIFT>(a, b)
}
#[inline(always)]
fn swizzle_dyn_within_blocks_i8x16(self, a: i8x16<Self>, indices: u8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, indices: u8x16<Avx512>) -> i8x16<Avx512> {
let bytes = token.cvt_to_bytes_i8x16(a).val.0;
let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into());
token.cvt_from_bytes_i8x16(u8x16 {
val: crate::support::Aligned128(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
_mm_add_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
_mm_sub_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
let dst_even = _mm_mullo_epi16(a.into(), b.into());
let dst_odd =
_mm_mullo_epi16(_mm_srli_epi16::<8>(a.into()), _mm_srli_epi16::<8>(b.into()));
_mm_or_si128(
_mm_slli_epi16(dst_odd, 8),
_mm_and_si128(dst_even, _mm_set1_epi16(0xFF)),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
_mm_and_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
_mm_or_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
_mm_xor_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
a ^ !0
}
#[inline(always)]
fn shl_i8x16(self, a: i8x16<Self>, shift: u32) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, shift: u32) -> i8x16<Avx512> {
let val = a.into();
let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
let lo_16 = _mm_unpacklo_epi8(val, _mm_cmpgt_epi8(_mm_setzero_si128(), val));
let hi_16 = _mm_unpackhi_epi8(val, _mm_cmpgt_epi8(_mm_setzero_si128(), val));
let lo_shifted = _mm_sll_epi16(lo_16, shift_count);
let hi_shifted = _mm_sll_epi16(hi_16, shift_count);
_mm_packs_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
let val = a.into();
let counts = b.into();
let zero = _mm_setzero_si128();
let value_extend = zero;
let lo_values = _mm_unpacklo_epi8(val, value_extend);
let hi_values = _mm_unpackhi_epi8(val, value_extend);
let lo_counts = _mm_unpacklo_epi8(counts, zero);
let hi_counts = _mm_unpackhi_epi8(counts, zero);
let byte_mask = _mm_set1_epi16(0x00ff);
let lo_shifted = _mm_and_si128(_mm_sllv_epi16(lo_values, lo_counts), byte_mask);
let hi_shifted = _mm_and_si128(_mm_sllv_epi16(hi_values, hi_counts), byte_mask);
_mm_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_i8x16(self, a: i8x16<Self>, shift: u32) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, shift: u32) -> i8x16<Avx512> {
let val = a.into();
let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
let lo_16 = _mm_unpacklo_epi8(val, _mm_cmpgt_epi8(_mm_setzero_si128(), val));
let hi_16 = _mm_unpackhi_epi8(val, _mm_cmpgt_epi8(_mm_setzero_si128(), val));
let lo_shifted = _mm_sra_epi16(lo_16, shift_count);
let hi_shifted = _mm_sra_epi16(hi_16, shift_count);
_mm_packs_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
let val = a.into();
let counts = b.into();
let zero = _mm_setzero_si128();
let value_extend = _mm_cmpgt_epi8(zero, val);
let lo_values = _mm_unpacklo_epi8(val, value_extend);
let hi_values = _mm_unpackhi_epi8(val, value_extend);
let lo_counts = _mm_unpacklo_epi8(counts, zero);
let hi_counts = _mm_unpackhi_epi8(counts, zero);
let byte_mask = _mm_set1_epi16(0x00ff);
let lo_shifted = _mm_and_si128(_mm_srav_epi16(lo_values, lo_counts), byte_mask);
let hi_shifted = _mm_and_si128(_mm_srav_epi16(hi_values, hi_counts), byte_mask);
_mm_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> mask8x16<Avx512> {
mask8x16 {
val: _mm_cmpeq_epi8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> mask8x16<Avx512> {
mask8x16 {
val: _mm_cmplt_epi8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> mask8x16<Avx512> {
mask8x16 {
val: _mm_cmple_epi8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> mask8x16<Avx512> {
mask8x16 {
val: _mm_cmpge_epi8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> mask8x16<Avx512> {
mask8x16 {
val: _mm_cmpgt_epi8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
_mm_unpacklo_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
_mm_unpackhi_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
_mm_permutex2var_epi8(
a.into(),
_mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
_mm_permutex2var_epi8(
a.into(),
_mm_setr_epi8(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i8x16<Self>, i8x16<Self>) {
(self.zip_low_i8x16(a, b), self.zip_high_i8x16(a, b))
}
#[inline(always)]
fn deinterleave_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i8x16<Self>, i8x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i8x16<Avx512>,
b: i8x16<Avx512>,
) -> (i8x16<Avx512>, i8x16<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm_permutex2var_epi8(
a,
_mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30),
b,
)
.simd_into(token),
_mm_permutex2var_epi8(
a,
_mm_setr_epi8(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_i8x16(self, a: mask8x16<Self>, b: i8x16<Self>, c: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask8x16<Avx512>,
b: i8x16<Avx512>,
c: i8x16<Avx512>,
) -> i8x16<Avx512> {
_mm_mask_blend_epi8(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn min_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
_mm_min_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x16<Avx512> {
_mm_max_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn combine_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>, b: i8x16<Avx512>) -> i8x32<Avx512> {
_mm256_setr_m128i(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn neg_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>) -> i8x16<Avx512> {
_mm_sub_epi8(_mm_setzero_si128(), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u8_i8x16(self, a: i8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>) -> u8x16<Avx512> {
__m128i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u32_i8x16(self, a: i8x16<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x16<Avx512>) -> u32x4<Avx512> {
__m128i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_u8x16(self, val: u8) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: u8) -> u8x16<Avx512> {
_mm_set1_epi8(val.cast_signed()).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_u8x16(self, val: [u8; 16usize]) -> u8x16<Self> {
u8x16 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_u8x16(self, val: &[u8; 16usize]) -> u8x16<Self> {
u8x16 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_u8x16(self, a: u8x16<Self>) -> [u8; 16usize] {
crate::transmute::checked_transmute_copy::<__m128i, [u8; 16usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_u8x16(self, a: &u8x16<Self>) -> &[u8; 16usize] {
crate::transmute::checked_cast_ref::<__m128i, [u8; 16usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_u8x16(self, a: &mut u8x16<Self>) -> &mut [u8; 16usize] {
crate::transmute::checked_cast_mut::<__m128i, [u8; 16usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_u8x16(self, a: u8x16<Self>, dest: &mut [u8; 16usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
u8x16 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
u8x16 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_u8x16<const SHIFT: usize>(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = dyn_alignr_128(
self,
self.cvt_to_bytes_u8x16(b).val.0,
self.cvt_to_bytes_u8x16(a).val.0,
SHIFT,
);
self.cvt_from_bytes_u8x16(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn slide_within_blocks_u8x16<const SHIFT: usize>(
self,
a: u8x16<Self>,
b: u8x16<Self>,
) -> u8x16<Self> {
self.slide_u8x16::<SHIFT>(a, b)
}
#[inline(always)]
fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, indices: u8x16<Avx512>) -> u8x16<Avx512> {
let bytes = token.cvt_to_bytes_u8x16(a).val.0;
let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into());
token.cvt_from_bytes_u8x16(u8x16 {
val: crate::support::Aligned128(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
_mm_add_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
_mm_sub_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
let dst_even = _mm_mullo_epi16(a.into(), b.into());
let dst_odd =
_mm_mullo_epi16(_mm_srli_epi16::<8>(a.into()), _mm_srli_epi16::<8>(b.into()));
_mm_or_si128(
_mm_slli_epi16(dst_odd, 8),
_mm_and_si128(dst_even, _mm_set1_epi16(0xFF)),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
_mm_and_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
_mm_or_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
_mm_xor_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
a ^ !0
}
#[inline(always)]
fn shl_u8x16(self, a: u8x16<Self>, shift: u32) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, shift: u32) -> u8x16<Avx512> {
let val = a.into();
let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
let lo_16 = _mm_unpacklo_epi8(val, _mm_setzero_si128());
let hi_16 = _mm_unpackhi_epi8(val, _mm_setzero_si128());
let lo_shifted = _mm_sll_epi16(lo_16, shift_count);
let hi_shifted = _mm_sll_epi16(hi_16, shift_count);
_mm_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
let val = a.into();
let counts = b.into();
let zero = _mm_setzero_si128();
let value_extend = zero;
let lo_values = _mm_unpacklo_epi8(val, value_extend);
let hi_values = _mm_unpackhi_epi8(val, value_extend);
let lo_counts = _mm_unpacklo_epi8(counts, zero);
let hi_counts = _mm_unpackhi_epi8(counts, zero);
let byte_mask = _mm_set1_epi16(0x00ff);
let lo_shifted = _mm_and_si128(_mm_sllv_epi16(lo_values, lo_counts), byte_mask);
let hi_shifted = _mm_and_si128(_mm_sllv_epi16(hi_values, hi_counts), byte_mask);
_mm_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_u8x16(self, a: u8x16<Self>, shift: u32) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, shift: u32) -> u8x16<Avx512> {
let val = a.into();
let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
let lo_16 = _mm_unpacklo_epi8(val, _mm_setzero_si128());
let hi_16 = _mm_unpackhi_epi8(val, _mm_setzero_si128());
let lo_shifted = _mm_srl_epi16(lo_16, shift_count);
let hi_shifted = _mm_srl_epi16(hi_16, shift_count);
_mm_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
let val = a.into();
let counts = b.into();
let zero = _mm_setzero_si128();
let value_extend = zero;
let lo_values = _mm_unpacklo_epi8(val, value_extend);
let hi_values = _mm_unpackhi_epi8(val, value_extend);
let lo_counts = _mm_unpacklo_epi8(counts, zero);
let hi_counts = _mm_unpackhi_epi8(counts, zero);
let byte_mask = _mm_set1_epi16(0x00ff);
let lo_shifted = _mm_and_si128(_mm_srlv_epi16(lo_values, lo_counts), byte_mask);
let hi_shifted = _mm_and_si128(_mm_srlv_epi16(hi_values, hi_counts), byte_mask);
_mm_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> mask8x16<Avx512> {
mask8x16 {
val: _mm_cmpeq_epu8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> mask8x16<Avx512> {
mask8x16 {
val: _mm_cmplt_epu8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> mask8x16<Avx512> {
mask8x16 {
val: _mm_cmple_epu8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> mask8x16<Avx512> {
mask8x16 {
val: _mm_cmpge_epu8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> mask8x16<Avx512> {
mask8x16 {
val: _mm_cmpgt_epu8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
_mm_unpacklo_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
_mm_unpackhi_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
_mm_permutex2var_epi8(
a.into(),
_mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
_mm_permutex2var_epi8(
a.into(),
_mm_setr_epi8(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u8x16<Self>, u8x16<Self>) {
(self.zip_low_u8x16(a, b), self.zip_high_u8x16(a, b))
}
#[inline(always)]
fn deinterleave_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u8x16<Self>, u8x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u8x16<Avx512>,
b: u8x16<Avx512>,
) -> (u8x16<Avx512>, u8x16<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm_permutex2var_epi8(
a,
_mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30),
b,
)
.simd_into(token),
_mm_permutex2var_epi8(
a,
_mm_setr_epi8(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_u8x16(self, a: mask8x16<Self>, b: u8x16<Self>, c: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask8x16<Avx512>,
b: u8x16<Avx512>,
c: u8x16<Avx512>,
) -> u8x16<Avx512> {
_mm_mask_blend_epi8(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn min_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
_mm_min_epu8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x16<Avx512> {
_mm_max_epu8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn combine_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>, b: u8x16<Avx512>) -> u8x32<Avx512> {
_mm256_setr_m128i(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn widen_u8x16(self, a: u8x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>) -> u16x16<Avx512> {
_mm256_cvtepu8_epi16(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u32_u8x16(self, a: u8x16<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x16<Avx512>) -> u32x4<Avx512> {
__m128i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_mask8x16(self, val: bool) -> mask8x16<Self> {
mask8x16 {
val: (if val { 65535u64 } else { 0 }) as _,
simd: self,
}
}
#[inline(always)]
fn load_array_mask8x16(self, val: [i8; 16usize]) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: [i8; 16usize]) -> mask8x16<Avx512> {
let lanes = crate::transmute::checked_transmute_copy(&val);
mask8x16 {
val: _mm_movepi8_mask(lanes),
simd: token,
}
}
);
kernel(self, val)
}
#[inline(always)]
fn as_array_mask8x16(self, a: mask8x16<Self>) -> [i8; 16usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: mask8x16<Avx512>) -> [i8; 16usize] {
let lanes = _mm_movm_epi8(a.val);
crate::transmute::checked_transmute_copy(&lanes)
}
);
kernel(self, a)
}
#[inline(always)]
fn from_bitmask_mask8x16(self, bits: u64) -> mask8x16<Self> {
mask8x16 {
val: (bits & 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn to_bitmask_mask8x16(self, a: mask8x16<Self>) -> u64 {
u64::from((a).val) & 65535u64
}
#[inline(always)]
fn set_mask8x16(self, a: &mut mask8x16<Self>, index: usize, value: bool) -> () {
assert!(
index < 16usize,
"mask lane index {index} is out of bounds for {} lanes",
16usize
);
let bit = 1u64 << index;
let bits = u64::from((a).val);
let bits = if value { bits | bit } else { bits & !bit };
*a = mask8x16 {
val: (bits) as _,
simd: self,
};
}
#[inline(always)]
fn and_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
mask8x16 {
val: ((u64::from((a).val) & u64::from((b).val)) & 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn or_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
mask8x16 {
val: ((u64::from((a).val) | u64::from((b).val)) & 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn xor_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
mask8x16 {
val: ((u64::from((a).val) ^ u64::from((b).val)) & 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn not_mask8x16(self, a: mask8x16<Self>) -> mask8x16<Self> {
mask8x16 {
val: ((!u64::from((a).val)) & 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn select_mask8x16(
self,
a: mask8x16<Self>,
b: mask8x16<Self>,
c: mask8x16<Self>,
) -> mask8x16<Self> {
mask8x16 {
val: (((u64::from((a).val) & u64::from((b).val))
| ((!u64::from((a).val)) & u64::from((c).val)))
& 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn simd_eq_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
mask8x16 {
val: (!u64::from(a.val ^ b.val) & 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn any_true_mask8x16(self, a: mask8x16<Self>) -> bool {
let bits = u64::from((a).val) & 65535u64;
bits != 0
}
#[inline(always)]
fn all_true_mask8x16(self, a: mask8x16<Self>) -> bool {
let bits = u64::from((a).val) & 65535u64;
bits == 65535u64
}
#[inline(always)]
fn any_false_mask8x16(self, a: mask8x16<Self>) -> bool {
let bits = u64::from((a).val) & 65535u64;
bits != 65535u64
}
#[inline(always)]
fn all_false_mask8x16(self, a: mask8x16<Self>) -> bool {
let bits = u64::from((a).val) & 65535u64;
bits == 0
}
#[inline(always)]
fn combine_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x32<Self> {
let bits = (u64::from(a.val) | (u64::from(b.val) << 16usize)) & 4294967295u64;
mask8x32 {
val: bits as _,
simd: self,
}
}
#[inline(always)]
fn splat_i16x8(self, val: i16) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: i16) -> i16x8<Avx512> {
_mm_set1_epi16(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_i16x8(self, val: [i16; 8usize]) -> i16x8<Self> {
i16x8 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_i16x8(self, val: &[i16; 8usize]) -> i16x8<Self> {
i16x8 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_i16x8(self, a: i16x8<Self>) -> [i16; 8usize] {
crate::transmute::checked_transmute_copy::<__m128i, [i16; 8usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_i16x8(self, a: &i16x8<Self>) -> &[i16; 8usize] {
crate::transmute::checked_cast_ref::<__m128i, [i16; 8usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_i16x8(self, a: &mut i16x8<Self>) -> &mut [i16; 8usize] {
crate::transmute::checked_cast_mut::<__m128i, [i16; 8usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_i16x8(self, a: i16x8<Self>, dest: &mut [i16; 8usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_i16x8(self, a: u8x16<Self>) -> i16x8<Self> {
i16x8 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_i16x8(self, a: i16x8<Self>) -> u8x16<Self> {
u8x16 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_i16x8<const SHIFT: usize>(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = dyn_alignr_128(
self,
self.cvt_to_bytes_i16x8(b).val.0,
self.cvt_to_bytes_i16x8(a).val.0,
SHIFT * 2usize,
);
self.cvt_from_bytes_i16x8(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn slide_within_blocks_i16x8<const SHIFT: usize>(
self,
a: i16x8<Self>,
b: i16x8<Self>,
) -> i16x8<Self> {
self.slide_i16x8::<SHIFT>(a, b)
}
#[inline(always)]
fn swizzle_dyn_within_blocks_i16x8(self, a: i16x8<Self>, indices: u8x16<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, indices: u8x16<Avx512>) -> i16x8<Avx512> {
let bytes = token.cvt_to_bytes_i16x8(a).val.0;
let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into());
token.cvt_from_bytes_i16x8(u8x16 {
val: crate::support::Aligned128(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
_mm_add_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
_mm_sub_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
_mm_mullo_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
_mm_and_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
_mm_or_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
_mm_xor_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
a ^ !0
}
#[inline(always)]
fn shl_i16x8(self, a: i16x8<Self>, shift: u32) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, shift: u32) -> i16x8<Avx512> {
_mm_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
_mm_sllv_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_i16x8(self, a: i16x8<Self>, shift: u32) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, shift: u32) -> i16x8<Avx512> {
_mm_sra_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
_mm_srav_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> mask16x8<Avx512> {
mask16x8 {
val: _mm_cmpeq_epi16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> mask16x8<Avx512> {
mask16x8 {
val: _mm_cmplt_epi16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> mask16x8<Avx512> {
mask16x8 {
val: _mm_cmple_epi16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> mask16x8<Avx512> {
mask16x8 {
val: _mm_cmpge_epi16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> mask16x8<Avx512> {
mask16x8 {
val: _mm_cmpgt_epi16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
_mm_unpacklo_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
_mm_unpackhi_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
_mm_permutex2var_epi16(
a.into(),
_mm_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
_mm_permutex2var_epi16(
a.into(),
_mm_setr_epi16(1, 3, 5, 7, 9, 11, 13, 15),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i16x8<Self>, i16x8<Self>) {
(self.zip_low_i16x8(a, b), self.zip_high_i16x8(a, b))
}
#[inline(always)]
fn deinterleave_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i16x8<Self>, i16x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i16x8<Avx512>,
b: i16x8<Avx512>,
) -> (i16x8<Avx512>, i16x8<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm_permutex2var_epi16(a, _mm_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14), b)
.simd_into(token),
_mm_permutex2var_epi16(a, _mm_setr_epi16(1, 3, 5, 7, 9, 11, 13, 15), b)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_i16x8(self, a: mask16x8<Self>, b: i16x8<Self>, c: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask16x8<Avx512>,
b: i16x8<Avx512>,
c: i16x8<Avx512>,
) -> i16x8<Avx512> {
_mm_mask_blend_epi16(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn min_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
_mm_min_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x8<Avx512> {
_mm_max_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn combine_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>, b: i16x8<Avx512>) -> i16x16<Avx512> {
_mm256_setr_m128i(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn neg_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>) -> i16x8<Avx512> {
_mm_sub_epi16(_mm_setzero_si128(), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u8_i16x8(self, a: i16x8<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>) -> u8x16<Avx512> {
__m128i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u32_i16x8(self, a: i16x8<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x8<Avx512>) -> u32x4<Avx512> {
__m128i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_u16x8(self, val: u16) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: u16) -> u16x8<Avx512> {
_mm_set1_epi16(val.cast_signed()).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_u16x8(self, val: [u16; 8usize]) -> u16x8<Self> {
u16x8 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_u16x8(self, val: &[u16; 8usize]) -> u16x8<Self> {
u16x8 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_u16x8(self, a: u16x8<Self>) -> [u16; 8usize] {
crate::transmute::checked_transmute_copy::<__m128i, [u16; 8usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_u16x8(self, a: &u16x8<Self>) -> &[u16; 8usize] {
crate::transmute::checked_cast_ref::<__m128i, [u16; 8usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_u16x8(self, a: &mut u16x8<Self>) -> &mut [u16; 8usize] {
crate::transmute::checked_cast_mut::<__m128i, [u16; 8usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_u16x8(self, a: u16x8<Self>, dest: &mut [u16; 8usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_u16x8(self, a: u8x16<Self>) -> u16x8<Self> {
u16x8 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_u16x8(self, a: u16x8<Self>) -> u8x16<Self> {
u8x16 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_u16x8<const SHIFT: usize>(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = dyn_alignr_128(
self,
self.cvt_to_bytes_u16x8(b).val.0,
self.cvt_to_bytes_u16x8(a).val.0,
SHIFT * 2usize,
);
self.cvt_from_bytes_u16x8(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn slide_within_blocks_u16x8<const SHIFT: usize>(
self,
a: u16x8<Self>,
b: u16x8<Self>,
) -> u16x8<Self> {
self.slide_u16x8::<SHIFT>(a, b)
}
#[inline(always)]
fn swizzle_dyn_within_blocks_u16x8(self, a: u16x8<Self>, indices: u8x16<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, indices: u8x16<Avx512>) -> u16x8<Avx512> {
let bytes = token.cvt_to_bytes_u16x8(a).val.0;
let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into());
token.cvt_from_bytes_u16x8(u8x16 {
val: crate::support::Aligned128(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
_mm_add_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
_mm_sub_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
_mm_mullo_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
_mm_and_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
_mm_or_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
_mm_xor_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
a ^ !0
}
#[inline(always)]
fn shl_u16x8(self, a: u16x8<Self>, shift: u32) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, shift: u32) -> u16x8<Avx512> {
_mm_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
_mm_sllv_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_u16x8(self, a: u16x8<Self>, shift: u32) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, shift: u32) -> u16x8<Avx512> {
_mm_srl_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
_mm_srlv_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> mask16x8<Avx512> {
mask16x8 {
val: _mm_cmpeq_epu16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> mask16x8<Avx512> {
mask16x8 {
val: _mm_cmplt_epu16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> mask16x8<Avx512> {
mask16x8 {
val: _mm_cmple_epu16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> mask16x8<Avx512> {
mask16x8 {
val: _mm_cmpge_epu16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> mask16x8<Avx512> {
mask16x8 {
val: _mm_cmpgt_epu16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
_mm_unpacklo_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
_mm_unpackhi_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
_mm_permutex2var_epi16(
a.into(),
_mm_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
_mm_permutex2var_epi16(
a.into(),
_mm_setr_epi16(1, 3, 5, 7, 9, 11, 13, 15),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u16x8<Self>, u16x8<Self>) {
(self.zip_low_u16x8(a, b), self.zip_high_u16x8(a, b))
}
#[inline(always)]
fn deinterleave_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u16x8<Self>, u16x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u16x8<Avx512>,
b: u16x8<Avx512>,
) -> (u16x8<Avx512>, u16x8<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm_permutex2var_epi16(a, _mm_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14), b)
.simd_into(token),
_mm_permutex2var_epi16(a, _mm_setr_epi16(1, 3, 5, 7, 9, 11, 13, 15), b)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_u16x8(self, a: mask16x8<Self>, b: u16x8<Self>, c: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask16x8<Avx512>,
b: u16x8<Avx512>,
c: u16x8<Avx512>,
) -> u16x8<Avx512> {
_mm_mask_blend_epi16(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn min_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
_mm_min_epu16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x8<Avx512> {
_mm_max_epu16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn combine_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>, b: u16x8<Avx512>) -> u16x16<Avx512> {
_mm256_setr_m128i(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn reinterpret_u8_u16x8(self, a: u16x8<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>) -> u8x16<Avx512> {
__m128i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u32_u16x8(self, a: u16x8<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x8<Avx512>) -> u32x4<Avx512> {
__m128i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_mask16x8(self, val: bool) -> mask16x8<Self> {
mask16x8 {
val: (if val { 255u64 } else { 0 }) as _,
simd: self,
}
}
#[inline(always)]
fn load_array_mask16x8(self, val: [i16; 8usize]) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: [i16; 8usize]) -> mask16x8<Avx512> {
let lanes = crate::transmute::checked_transmute_copy(&val);
mask16x8 {
val: _mm_movepi16_mask(lanes),
simd: token,
}
}
);
kernel(self, val)
}
#[inline(always)]
fn as_array_mask16x8(self, a: mask16x8<Self>) -> [i16; 8usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: mask16x8<Avx512>) -> [i16; 8usize] {
let lanes = _mm_movm_epi16(a.val);
crate::transmute::checked_transmute_copy(&lanes)
}
);
kernel(self, a)
}
#[inline(always)]
fn from_bitmask_mask16x8(self, bits: u64) -> mask16x8<Self> {
mask16x8 {
val: (bits & 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn to_bitmask_mask16x8(self, a: mask16x8<Self>) -> u64 {
u64::from((a).val) & 255u64
}
#[inline(always)]
fn set_mask16x8(self, a: &mut mask16x8<Self>, index: usize, value: bool) -> () {
assert!(
index < 8usize,
"mask lane index {index} is out of bounds for {} lanes",
8usize
);
let bit = 1u64 << index;
let bits = u64::from((a).val);
let bits = if value { bits | bit } else { bits & !bit };
*a = mask16x8 {
val: (bits) as _,
simd: self,
};
}
#[inline(always)]
fn and_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
mask16x8 {
val: ((u64::from((a).val) & u64::from((b).val)) & 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn or_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
mask16x8 {
val: ((u64::from((a).val) | u64::from((b).val)) & 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn xor_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
mask16x8 {
val: ((u64::from((a).val) ^ u64::from((b).val)) & 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn not_mask16x8(self, a: mask16x8<Self>) -> mask16x8<Self> {
mask16x8 {
val: ((!u64::from((a).val)) & 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn select_mask16x8(
self,
a: mask16x8<Self>,
b: mask16x8<Self>,
c: mask16x8<Self>,
) -> mask16x8<Self> {
mask16x8 {
val: (((u64::from((a).val) & u64::from((b).val))
| ((!u64::from((a).val)) & u64::from((c).val)))
& 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn simd_eq_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
mask16x8 {
val: (!u64::from(a.val ^ b.val) & 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn any_true_mask16x8(self, a: mask16x8<Self>) -> bool {
let bits = u64::from((a).val) & 255u64;
bits != 0
}
#[inline(always)]
fn all_true_mask16x8(self, a: mask16x8<Self>) -> bool {
let bits = u64::from((a).val) & 255u64;
bits == 255u64
}
#[inline(always)]
fn any_false_mask16x8(self, a: mask16x8<Self>) -> bool {
let bits = u64::from((a).val) & 255u64;
bits != 255u64
}
#[inline(always)]
fn all_false_mask16x8(self, a: mask16x8<Self>) -> bool {
let bits = u64::from((a).val) & 255u64;
bits == 0
}
#[inline(always)]
fn combine_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x16<Self> {
let bits = (u64::from(a.val) | (u64::from(b.val) << 8usize)) & 65535u64;
mask16x16 {
val: bits as _,
simd: self,
}
}
#[inline(always)]
fn splat_i32x4(self, val: i32) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: i32) -> i32x4<Avx512> {
_mm_set1_epi32(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_i32x4(self, val: [i32; 4usize]) -> i32x4<Self> {
i32x4 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_i32x4(self, val: &[i32; 4usize]) -> i32x4<Self> {
i32x4 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_i32x4(self, a: i32x4<Self>) -> [i32; 4usize] {
crate::transmute::checked_transmute_copy::<__m128i, [i32; 4usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_i32x4(self, a: &i32x4<Self>) -> &[i32; 4usize] {
crate::transmute::checked_cast_ref::<__m128i, [i32; 4usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_i32x4(self, a: &mut i32x4<Self>) -> &mut [i32; 4usize] {
crate::transmute::checked_cast_mut::<__m128i, [i32; 4usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_i32x4(self, a: i32x4<Self>, dest: &mut [i32; 4usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_i32x4(self, a: u8x16<Self>) -> i32x4<Self> {
i32x4 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_i32x4(self, a: i32x4<Self>) -> u8x16<Self> {
u8x16 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_i32x4<const SHIFT: usize>(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = dyn_alignr_128(
self,
self.cvt_to_bytes_i32x4(b).val.0,
self.cvt_to_bytes_i32x4(a).val.0,
SHIFT * 4usize,
);
self.cvt_from_bytes_i32x4(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn slide_within_blocks_i32x4<const SHIFT: usize>(
self,
a: i32x4<Self>,
b: i32x4<Self>,
) -> i32x4<Self> {
self.slide_i32x4::<SHIFT>(a, b)
}
#[inline(always)]
fn swizzle_dyn_within_blocks_i32x4(self, a: i32x4<Self>, indices: u8x16<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, indices: u8x16<Avx512>) -> i32x4<Avx512> {
let bytes = token.cvt_to_bytes_i32x4(a).val.0;
let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into());
token.cvt_from_bytes_i32x4(u8x16 {
val: crate::support::Aligned128(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
_mm_add_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
_mm_sub_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
_mm_mullo_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
_mm_and_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
_mm_or_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
_mm_xor_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
a ^ !0
}
#[inline(always)]
fn shl_i32x4(self, a: i32x4<Self>, shift: u32) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, shift: u32) -> i32x4<Avx512> {
_mm_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
_mm_sllv_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_i32x4(self, a: i32x4<Self>, shift: u32) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, shift: u32) -> i32x4<Avx512> {
_mm_sra_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
_mm_srav_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> mask32x4<Avx512> {
mask32x4 {
val: _mm_cmpeq_epi32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> mask32x4<Avx512> {
mask32x4 {
val: _mm_cmplt_epi32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> mask32x4<Avx512> {
mask32x4 {
val: _mm_cmple_epi32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> mask32x4<Avx512> {
mask32x4 {
val: _mm_cmpge_epi32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> mask32x4<Avx512> {
mask32x4 {
val: _mm_cmpgt_epi32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
_mm_unpacklo_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
_mm_unpackhi_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
_mm_permutex2var_epi32(a.into(), _mm_setr_epi32(0, 2, 4, 6), b.into())
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
_mm_permutex2var_epi32(a.into(), _mm_setr_epi32(1, 3, 5, 7), b.into())
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i32x4<Self>, i32x4<Self>) {
(self.zip_low_i32x4(a, b), self.zip_high_i32x4(a, b))
}
#[inline(always)]
fn deinterleave_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i32x4<Self>, i32x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i32x4<Avx512>,
b: i32x4<Avx512>,
) -> (i32x4<Avx512>, i32x4<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm_permutex2var_epi32(a, _mm_setr_epi32(0, 2, 4, 6), b).simd_into(token),
_mm_permutex2var_epi32(a, _mm_setr_epi32(1, 3, 5, 7), b).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_i32x4(self, a: mask32x4<Self>, b: i32x4<Self>, c: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask32x4<Avx512>,
b: i32x4<Avx512>,
c: i32x4<Avx512>,
) -> i32x4<Avx512> {
_mm_mask_blend_epi32(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn min_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
_mm_min_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x4<Avx512> {
_mm_max_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn combine_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>, b: i32x4<Avx512>) -> i32x8<Avx512> {
_mm256_setr_m128i(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn neg_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>) -> i32x4<Avx512> {
_mm_sub_epi32(_mm_setzero_si128(), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u8_i32x4(self, a: i32x4<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>) -> u8x16<Avx512> {
__m128i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u32_i32x4(self, a: i32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>) -> u32x4<Avx512> {
__m128i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_f32_i32x4(self, a: i32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x4<Avx512>) -> f32x4<Avx512> {
_mm_cvtepi32_ps(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_u32x4(self, val: u32) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: u32) -> u32x4<Avx512> {
_mm_set1_epi32(val.cast_signed()).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_u32x4(self, val: [u32; 4usize]) -> u32x4<Self> {
u32x4 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_u32x4(self, val: &[u32; 4usize]) -> u32x4<Self> {
u32x4 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_u32x4(self, a: u32x4<Self>) -> [u32; 4usize] {
crate::transmute::checked_transmute_copy::<__m128i, [u32; 4usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_u32x4(self, a: &u32x4<Self>) -> &[u32; 4usize] {
crate::transmute::checked_cast_ref::<__m128i, [u32; 4usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_u32x4(self, a: &mut u32x4<Self>) -> &mut [u32; 4usize] {
crate::transmute::checked_cast_mut::<__m128i, [u32; 4usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_u32x4(self, a: u32x4<Self>, dest: &mut [u32; 4usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_u32x4(self, a: u8x16<Self>) -> u32x4<Self> {
u32x4 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_u32x4(self, a: u32x4<Self>) -> u8x16<Self> {
u8x16 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_u32x4<const SHIFT: usize>(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = dyn_alignr_128(
self,
self.cvt_to_bytes_u32x4(b).val.0,
self.cvt_to_bytes_u32x4(a).val.0,
SHIFT * 4usize,
);
self.cvt_from_bytes_u32x4(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn slide_within_blocks_u32x4<const SHIFT: usize>(
self,
a: u32x4<Self>,
b: u32x4<Self>,
) -> u32x4<Self> {
self.slide_u32x4::<SHIFT>(a, b)
}
#[inline(always)]
fn swizzle_dyn_within_blocks_u32x4(self, a: u32x4<Self>, indices: u8x16<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, indices: u8x16<Avx512>) -> u32x4<Avx512> {
let bytes = token.cvt_to_bytes_u32x4(a).val.0;
let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into());
token.cvt_from_bytes_u32x4(u8x16 {
val: crate::support::Aligned128(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
_mm_add_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
_mm_sub_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
_mm_mullo_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
_mm_and_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
_mm_or_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
_mm_xor_si128(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
a ^ !0
}
#[inline(always)]
fn shl_u32x4(self, a: u32x4<Self>, shift: u32) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, shift: u32) -> u32x4<Avx512> {
_mm_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
_mm_sllv_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_u32x4(self, a: u32x4<Self>, shift: u32) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, shift: u32) -> u32x4<Avx512> {
_mm_srl_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
_mm_srlv_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> mask32x4<Avx512> {
mask32x4 {
val: _mm_cmpeq_epu32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> mask32x4<Avx512> {
mask32x4 {
val: _mm_cmplt_epu32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> mask32x4<Avx512> {
mask32x4 {
val: _mm_cmple_epu32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> mask32x4<Avx512> {
mask32x4 {
val: _mm_cmpge_epu32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> mask32x4<Avx512> {
mask32x4 {
val: _mm_cmpgt_epu32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
_mm_unpacklo_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
_mm_unpackhi_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
_mm_permutex2var_epi32(a.into(), _mm_setr_epi32(0, 2, 4, 6), b.into())
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
_mm_permutex2var_epi32(a.into(), _mm_setr_epi32(1, 3, 5, 7), b.into())
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u32x4<Self>, u32x4<Self>) {
(self.zip_low_u32x4(a, b), self.zip_high_u32x4(a, b))
}
#[inline(always)]
fn deinterleave_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u32x4<Self>, u32x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u32x4<Avx512>,
b: u32x4<Avx512>,
) -> (u32x4<Avx512>, u32x4<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm_permutex2var_epi32(a, _mm_setr_epi32(0, 2, 4, 6), b).simd_into(token),
_mm_permutex2var_epi32(a, _mm_setr_epi32(1, 3, 5, 7), b).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_u32x4(self, a: mask32x4<Self>, b: u32x4<Self>, c: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask32x4<Avx512>,
b: u32x4<Avx512>,
c: u32x4<Avx512>,
) -> u32x4<Avx512> {
_mm_mask_blend_epi32(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn min_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
_mm_min_epu32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x4<Avx512> {
_mm_max_epu32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn combine_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>, b: u32x4<Avx512>) -> u32x8<Avx512> {
_mm256_setr_m128i(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn reinterpret_u8_u32x4(self, a: u32x4<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>) -> u8x16<Avx512> {
__m128i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_f32_u32x4(self, a: u32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x4<Avx512>) -> f32x4<Avx512> {
_mm512_castps512_ps128(_mm512_cvtepu32_ps(_mm512_zextsi128_si512(a.into())))
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_mask32x4(self, val: bool) -> mask32x4<Self> {
mask32x4 {
val: (if val { 15u64 } else { 0 }) as _,
simd: self,
}
}
#[inline(always)]
fn load_array_mask32x4(self, val: [i32; 4usize]) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: [i32; 4usize]) -> mask32x4<Avx512> {
let lanes = crate::transmute::checked_transmute_copy(&val);
mask32x4 {
val: _mm_movepi32_mask(lanes),
simd: token,
}
}
);
kernel(self, val)
}
#[inline(always)]
fn as_array_mask32x4(self, a: mask32x4<Self>) -> [i32; 4usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: mask32x4<Avx512>) -> [i32; 4usize] {
let lanes = _mm_movm_epi32(a.val);
crate::transmute::checked_transmute_copy(&lanes)
}
);
kernel(self, a)
}
#[inline(always)]
fn from_bitmask_mask32x4(self, bits: u64) -> mask32x4<Self> {
mask32x4 {
val: (bits & 15u64) as _,
simd: self,
}
}
#[inline(always)]
fn to_bitmask_mask32x4(self, a: mask32x4<Self>) -> u64 {
u64::from((a).val) & 15u64
}
#[inline(always)]
fn set_mask32x4(self, a: &mut mask32x4<Self>, index: usize, value: bool) -> () {
assert!(
index < 4usize,
"mask lane index {index} is out of bounds for {} lanes",
4usize
);
let bit = 1u64 << index;
let bits = u64::from((a).val);
let bits = if value { bits | bit } else { bits & !bit };
*a = mask32x4 {
val: (bits) as _,
simd: self,
};
}
#[inline(always)]
fn and_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
mask32x4 {
val: ((u64::from((a).val) & u64::from((b).val)) & 15u64) as _,
simd: self,
}
}
#[inline(always)]
fn or_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
mask32x4 {
val: ((u64::from((a).val) | u64::from((b).val)) & 15u64) as _,
simd: self,
}
}
#[inline(always)]
fn xor_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
mask32x4 {
val: ((u64::from((a).val) ^ u64::from((b).val)) & 15u64) as _,
simd: self,
}
}
#[inline(always)]
fn not_mask32x4(self, a: mask32x4<Self>) -> mask32x4<Self> {
mask32x4 {
val: ((!u64::from((a).val)) & 15u64) as _,
simd: self,
}
}
#[inline(always)]
fn select_mask32x4(
self,
a: mask32x4<Self>,
b: mask32x4<Self>,
c: mask32x4<Self>,
) -> mask32x4<Self> {
mask32x4 {
val: (((u64::from((a).val) & u64::from((b).val))
| ((!u64::from((a).val)) & u64::from((c).val)))
& 15u64) as _,
simd: self,
}
}
#[inline(always)]
fn simd_eq_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
mask32x4 {
val: (!u64::from(a.val ^ b.val) & 15u64) as _,
simd: self,
}
}
#[inline(always)]
fn any_true_mask32x4(self, a: mask32x4<Self>) -> bool {
let bits = u64::from((a).val) & 15u64;
bits != 0
}
#[inline(always)]
fn all_true_mask32x4(self, a: mask32x4<Self>) -> bool {
let bits = u64::from((a).val) & 15u64;
bits == 15u64
}
#[inline(always)]
fn any_false_mask32x4(self, a: mask32x4<Self>) -> bool {
let bits = u64::from((a).val) & 15u64;
bits != 15u64
}
#[inline(always)]
fn all_false_mask32x4(self, a: mask32x4<Self>) -> bool {
let bits = u64::from((a).val) & 15u64;
bits == 0
}
#[inline(always)]
fn combine_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x8<Self> {
let bits = (u64::from(a.val) | (u64::from(b.val) << 4usize)) & 255u64;
mask32x8 {
val: bits as _,
simd: self,
}
}
#[inline(always)]
fn splat_f64x2(self, val: f64) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: f64) -> f64x2<Avx512> {
_mm_set1_pd(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_f64x2(self, val: [f64; 2usize]) -> f64x2<Self> {
f64x2 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_f64x2(self, val: &[f64; 2usize]) -> f64x2<Self> {
f64x2 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_f64x2(self, a: f64x2<Self>) -> [f64; 2usize] {
crate::transmute::checked_transmute_copy::<__m128d, [f64; 2usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_f64x2(self, a: &f64x2<Self>) -> &[f64; 2usize] {
crate::transmute::checked_cast_ref::<__m128d, [f64; 2usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_f64x2(self, a: &mut f64x2<Self>) -> &mut [f64; 2usize] {
crate::transmute::checked_cast_mut::<__m128d, [f64; 2usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_f64x2(self, a: f64x2<Self>, dest: &mut [f64; 2usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_f64x2(self, a: u8x16<Self>) -> f64x2<Self> {
f64x2 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_f64x2(self, a: f64x2<Self>) -> u8x16<Self> {
u8x16 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_f64x2<const SHIFT: usize>(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
if SHIFT >= 2usize {
return b;
}
let result = dyn_alignr_128(
self,
self.cvt_to_bytes_f64x2(b).val.0,
self.cvt_to_bytes_f64x2(a).val.0,
SHIFT * 8usize,
);
self.cvt_from_bytes_f64x2(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn slide_within_blocks_f64x2<const SHIFT: usize>(
self,
a: f64x2<Self>,
b: f64x2<Self>,
) -> f64x2<Self> {
self.slide_f64x2::<SHIFT>(a, b)
}
#[inline(always)]
fn swizzle_dyn_within_blocks_f64x2(self, a: f64x2<Self>, indices: u8x16<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>, indices: u8x16<Avx512>) -> f64x2<Avx512> {
let bytes = token.cvt_to_bytes_f64x2(a).val.0;
let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into());
token.cvt_from_bytes_f64x2(u8x16 {
val: crate::support::Aligned128(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn abs_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64x2<Avx512> {
_mm_andnot_pd(_mm_set1_pd(-0.0), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64x2<Avx512> {
_mm_xor_pd(a.into(), _mm_set1_pd(-0.0)).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn sqrt_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64x2<Avx512> {
_mm_sqrt_pd(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn approximate_recip_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64x2<Avx512> {
_mm_rcp14_pd(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn add_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
_mm_add_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
_mm_sub_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
_mm_mul_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn div_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
_mm_div_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn copysign_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
let mask = _mm_set1_pd(-0.0);
_mm_or_pd(_mm_and_pd(mask, b.into()), _mm_andnot_pd(mask, a.into()))
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> mask64x2<Avx512> {
mask64x2 {
val: _mm_cmp_pd_mask::<0i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> mask64x2<Avx512> {
mask64x2 {
val: _mm_cmp_pd_mask::<17i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> mask64x2<Avx512> {
mask64x2 {
val: _mm_cmp_pd_mask::<18i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> mask64x2<Avx512> {
mask64x2 {
val: _mm_cmp_pd_mask::<29i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> mask64x2<Avx512> {
mask64x2 {
val: _mm_cmp_pd_mask::<30i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
_mm_unpacklo_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
_mm_unpackhi_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
_mm_shuffle_pd::<0b00>(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
_mm_shuffle_pd::<0b11>(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> (f64x2<Self>, f64x2<Self>) {
(self.zip_low_f64x2(a, b), self.zip_high_f64x2(a, b))
}
#[inline(always)]
fn deinterleave_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> (f64x2<Self>, f64x2<Self>) {
(self.unzip_low_f64x2(a, b), self.unzip_high_f64x2(a, b))
}
#[inline(always)]
fn max_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
_mm_max_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
_mm_min_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
_mm_range_pd::<5i32>(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x2<Avx512> {
_mm_range_pd::<4i32>(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_add_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f64x2<Avx512>,
b: f64x2<Avx512>,
c: f64x2<Avx512>,
) -> f64x2<Avx512> {
_mm_fmadd_pd(a.into(), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn mul_sub_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f64x2<Avx512>,
b: f64x2<Avx512>,
c: f64x2<Avx512>,
) -> f64x2<Avx512> {
_mm_fmsub_pd(a.into(), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn floor_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64x2<Avx512> {
_mm_round_pd::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn ceil_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64x2<Avx512> {
_mm_round_pd::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn round_ties_even_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64x2<Avx512> {
_mm_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn fract_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
a - self.trunc_f64x2(a)
}
#[inline(always)]
fn trunc_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>) -> f64x2<Avx512> {
_mm_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn select_f64x2(self, a: mask64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask64x2<Avx512>,
b: f64x2<Avx512>,
c: f64x2<Avx512>,
) -> f64x2<Avx512> {
_mm_mask_blend_pd(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>, b: f64x2<Avx512>) -> f64x4<Avx512> {
_mm256_setr_m128d(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn reinterpret_f32_f64x2(self, a: f64x2<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x2<Avx512>) -> f32x4<Avx512> {
_mm_castpd_ps(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_mask64x2(self, val: bool) -> mask64x2<Self> {
mask64x2 {
val: (if val { 3u64 } else { 0 }) as _,
simd: self,
}
}
#[inline(always)]
fn load_array_mask64x2(self, val: [i64; 2usize]) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: [i64; 2usize]) -> mask64x2<Avx512> {
let lanes = crate::transmute::checked_transmute_copy(&val);
mask64x2 {
val: _mm_movepi64_mask(lanes),
simd: token,
}
}
);
kernel(self, val)
}
#[inline(always)]
fn as_array_mask64x2(self, a: mask64x2<Self>) -> [i64; 2usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: mask64x2<Avx512>) -> [i64; 2usize] {
let lanes = _mm_movm_epi64(a.val);
crate::transmute::checked_transmute_copy(&lanes)
}
);
kernel(self, a)
}
#[inline(always)]
fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2<Self> {
mask64x2 {
val: (bits & 3u64) as _,
simd: self,
}
}
#[inline(always)]
fn to_bitmask_mask64x2(self, a: mask64x2<Self>) -> u64 {
u64::from((a).val) & 3u64
}
#[inline(always)]
fn set_mask64x2(self, a: &mut mask64x2<Self>, index: usize, value: bool) -> () {
assert!(
index < 2usize,
"mask lane index {index} is out of bounds for {} lanes",
2usize
);
let bit = 1u64 << index;
let bits = u64::from((a).val);
let bits = if value { bits | bit } else { bits & !bit };
*a = mask64x2 {
val: (bits) as _,
simd: self,
};
}
#[inline(always)]
fn and_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
mask64x2 {
val: ((u64::from((a).val) & u64::from((b).val)) & 3u64) as _,
simd: self,
}
}
#[inline(always)]
fn or_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
mask64x2 {
val: ((u64::from((a).val) | u64::from((b).val)) & 3u64) as _,
simd: self,
}
}
#[inline(always)]
fn xor_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
mask64x2 {
val: ((u64::from((a).val) ^ u64::from((b).val)) & 3u64) as _,
simd: self,
}
}
#[inline(always)]
fn not_mask64x2(self, a: mask64x2<Self>) -> mask64x2<Self> {
mask64x2 {
val: ((!u64::from((a).val)) & 3u64) as _,
simd: self,
}
}
#[inline(always)]
fn select_mask64x2(
self,
a: mask64x2<Self>,
b: mask64x2<Self>,
c: mask64x2<Self>,
) -> mask64x2<Self> {
mask64x2 {
val: (((u64::from((a).val) & u64::from((b).val))
| ((!u64::from((a).val)) & u64::from((c).val)))
& 3u64) as _,
simd: self,
}
}
#[inline(always)]
fn simd_eq_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
mask64x2 {
val: (!u64::from(a.val ^ b.val) & 3u64) as _,
simd: self,
}
}
#[inline(always)]
fn any_true_mask64x2(self, a: mask64x2<Self>) -> bool {
let bits = u64::from((a).val) & 3u64;
bits != 0
}
#[inline(always)]
fn all_true_mask64x2(self, a: mask64x2<Self>) -> bool {
let bits = u64::from((a).val) & 3u64;
bits == 3u64
}
#[inline(always)]
fn any_false_mask64x2(self, a: mask64x2<Self>) -> bool {
let bits = u64::from((a).val) & 3u64;
bits != 3u64
}
#[inline(always)]
fn all_false_mask64x2(self, a: mask64x2<Self>) -> bool {
let bits = u64::from((a).val) & 3u64;
bits == 0
}
#[inline(always)]
fn combine_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x4<Self> {
let bits = (u64::from(a.val) | (u64::from(b.val) << 2usize)) & 15u64;
mask64x4 {
val: bits as _,
simd: self,
}
}
#[inline(always)]
fn splat_f32x8(self, val: f32) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: f32) -> f32x8<Avx512> {
_mm256_set1_ps(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_f32x8(self, val: [f32; 8usize]) -> f32x8<Self> {
f32x8 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_f32x8(self, val: &[f32; 8usize]) -> f32x8<Self> {
f32x8 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_f32x8(self, a: f32x8<Self>) -> [f32; 8usize] {
crate::transmute::checked_transmute_copy::<__m256, [f32; 8usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_f32x8(self, a: &f32x8<Self>) -> &[f32; 8usize] {
crate::transmute::checked_cast_ref::<__m256, [f32; 8usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_f32x8(self, a: &mut f32x8<Self>) -> &mut [f32; 8usize] {
crate::transmute::checked_cast_mut::<__m256, [f32; 8usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_f32x8(self, a: f32x8<Self>, dest: &mut [f32; 8usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_f32x8(self, a: u8x32<Self>) -> f32x8<Self> {
f32x8 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_f32x8(self, a: f32x8<Self>) -> u8x32<Self> {
u8x32 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_f32x8<const SHIFT: usize>(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f32x8<Avx512>,
b: f32x8<Avx512>,
shift: usize,
) -> f32x8<Avx512> {
if shift >= 8usize {
return b;
}
let idx = _mm256_add_epi8(
_mm256_setr_epi8(
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
),
_mm256_set1_epi8((shift * 4usize) as i8),
);
let result = _mm256_permutex2var_epi8(
token.cvt_to_bytes_f32x8(a).val.0,
idx,
token.cvt_to_bytes_f32x8(b).val.0,
);
token.cvt_from_bytes_f32x8(u8x32 {
val: crate::support::Aligned256(result),
simd: token,
})
}
);
kernel(self, a, b, SHIFT)
}
#[inline(always)]
fn slide_within_blocks_f32x8<const SHIFT: usize>(
self,
a: f32x8<Self>,
b: f32x8<Self>,
) -> f32x8<Self> {
if SHIFT == 0 {
return a;
}
if SHIFT >= 4usize {
return b;
}
let a = self.cvt_to_bytes_f32x8(a).val.0;
let b = self.cvt_to_bytes_f32x8(b).val.0;
let result = dyn_alignr_256(self, b, a, SHIFT * 4usize);
self.cvt_from_bytes_f32x8(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_within_blocks_f32x8(self, a: f32x8<Self>, indices: u8x32<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>, indices: u8x32<Avx512>) -> f32x8<Avx512> {
let bytes = token.cvt_to_bytes_f32x8(a).val.0;
let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into());
token.cvt_from_bytes_f32x8(u8x32 {
val: crate::support::Aligned256(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn abs_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>) -> f32x8<Avx512> {
_mm256_andnot_ps(_mm256_set1_ps(-0.0), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>) -> f32x8<Avx512> {
_mm256_xor_ps(a.into(), _mm256_set1_ps(-0.0)).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn sqrt_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>) -> f32x8<Avx512> {
_mm256_sqrt_ps(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn approximate_recip_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>) -> f32x8<Avx512> {
_mm256_rcp14_ps(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn add_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
_mm256_add_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
_mm256_sub_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
_mm256_mul_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn div_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
_mm256_div_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn copysign_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
let mask = _mm256_set1_ps(-0.0);
_mm256_or_ps(
_mm256_and_ps(mask, b.into()),
_mm256_andnot_ps(mask, a.into()),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> mask32x8<Avx512> {
mask32x8 {
val: _mm256_cmp_ps_mask::<0i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> mask32x8<Avx512> {
mask32x8 {
val: _mm256_cmp_ps_mask::<17i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> mask32x8<Avx512> {
mask32x8 {
val: _mm256_cmp_ps_mask::<18i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> mask32x8<Avx512> {
mask32x8 {
val: _mm256_cmp_ps_mask::<29i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> mask32x8<Avx512> {
mask32x8 {
val: _mm256_cmp_ps_mask::<30i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
_mm256_permutex2var_ps(
a.into(),
_mm256_setr_epi32(0, 8, 1, 9, 2, 10, 3, 11),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
_mm256_permutex2var_ps(
a.into(),
_mm256_setr_epi32(4, 12, 5, 13, 6, 14, 7, 15),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
_mm256_permutex2var_ps(
a.into(),
_mm256_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
_mm256_permutex2var_ps(
a.into(),
_mm256_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> (f32x8<Self>, f32x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f32x8<Avx512>,
b: f32x8<Avx512>,
) -> (f32x8<Avx512>, f32x8<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm256_permutex2var_ps(a, _mm256_setr_epi32(0, 8, 1, 9, 2, 10, 3, 11), b)
.simd_into(token),
_mm256_permutex2var_ps(a, _mm256_setr_epi32(4, 12, 5, 13, 6, 14, 7, 15), b)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> (f32x8<Self>, f32x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f32x8<Avx512>,
b: f32x8<Avx512>,
) -> (f32x8<Avx512>, f32x8<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm256_permutex2var_ps(a, _mm256_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14), b)
.simd_into(token),
_mm256_permutex2var_ps(a, _mm256_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15), b)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
_mm256_max_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
_mm256_min_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_precise_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
_mm256_range_ps::<5i32>(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_precise_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x8<Avx512> {
_mm256_range_ps::<4i32>(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_add_f32x8(self, a: f32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f32x8<Avx512>,
b: f32x8<Avx512>,
c: f32x8<Avx512>,
) -> f32x8<Avx512> {
_mm256_fmadd_ps(a.into(), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn mul_sub_f32x8(self, a: f32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f32x8<Avx512>,
b: f32x8<Avx512>,
c: f32x8<Avx512>,
) -> f32x8<Avx512> {
_mm256_fmsub_ps(a.into(), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn floor_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>) -> f32x8<Avx512> {
_mm256_round_ps::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn ceil_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>) -> f32x8<Avx512> {
_mm256_round_ps::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn round_ties_even_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>) -> f32x8<Avx512> {
_mm256_round_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn fract_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
a - self.trunc_f32x8(a)
}
#[inline(always)]
fn trunc_f32x8(self, a: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>) -> f32x8<Avx512> {
_mm256_round_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn select_f32x8(self, a: mask32x8<Self>, b: f32x8<Self>, c: f32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask32x8<Avx512>,
b: f32x8<Avx512>,
c: f32x8<Avx512>,
) -> f32x8<Avx512> {
_mm256_mask_blend_ps(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>, b: f32x8<Avx512>) -> f32x16<Avx512> {
_mm512_insertf32x8::<1>(_mm512_castps256_ps512(a.into()), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn split_f32x8(self, a: f32x8<Self>) -> (f32x4<Self>, f32x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>) -> (f32x4<Avx512>, f32x4<Avx512>) {
(
_mm256_extractf128_ps::<0>(a.into()).simd_into(token),
_mm256_extractf128_ps::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_f64_f32x8(self, a: f32x8<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>) -> f64x4<Avx512> {
_mm256_castps_pd(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_i32_f32x8(self, a: f32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>) -> i32x8<Avx512> {
_mm256_castps_si256(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u8_f32x8(self, a: f32x8<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>) -> u8x32<Avx512> {
_mm256_castps_si256(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u32_f32x8(self, a: f32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>) -> u32x8<Avx512> {
_mm256_castps_si256(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_u32_f32x8(self, a: f32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>) -> u32x8<Avx512> {
_mm256_cvttps_epu32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_u32_precise_f32x8(self, a: f32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>) -> u32x8<Avx512> {
let a = _mm256_max_ps(a.into(), _mm256_setzero_ps());
let mut converted = _mm256_cvttps_epu32(a);
let exceeds_unsigned_range =
_mm256_cmp_ps_mask::<17i32>(_mm256_set1_ps(4294967040.0), a);
converted = _mm256_mask_blend_epi32(
exceeds_unsigned_range,
converted,
_mm256_set1_epi32(u32::MAX.cast_signed()),
);
converted.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_i32_f32x8(self, a: f32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>) -> i32x8<Avx512> {
_mm256_cvttps_epi32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_i32_precise_f32x8(self, a: f32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x8<Avx512>) -> i32x8<Avx512> {
let a = a.into();
let in_range = _mm256_cmp_ps_mask::<17i32>(a, _mm256_set1_ps(2147483648.0));
let mut converted =
_mm256_mask_cvttps_epi32(_mm256_set1_epi32(i32::MAX), in_range, a);
let is_not_nan = _mm256_cmp_ps_mask::<7i32>(a, a);
converted = _mm256_mask_blend_epi32(is_not_nan, _mm256_setzero_si256(), converted);
converted.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_i8x32(self, val: i8) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: i8) -> i8x32<Avx512> {
_mm256_set1_epi8(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_i8x32(self, val: [i8; 32usize]) -> i8x32<Self> {
i8x32 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_i8x32(self, val: &[i8; 32usize]) -> i8x32<Self> {
i8x32 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_i8x32(self, a: i8x32<Self>) -> [i8; 32usize] {
crate::transmute::checked_transmute_copy::<__m256i, [i8; 32usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_i8x32(self, a: &i8x32<Self>) -> &[i8; 32usize] {
crate::transmute::checked_cast_ref::<__m256i, [i8; 32usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_i8x32(self, a: &mut i8x32<Self>) -> &mut [i8; 32usize] {
crate::transmute::checked_cast_mut::<__m256i, [i8; 32usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_i8x32(self, a: i8x32<Self>, dest: &mut [i8; 32usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_i8x32(self, a: u8x32<Self>) -> i8x32<Self> {
i8x32 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_i8x32(self, a: i8x32<Self>) -> u8x32<Self> {
u8x32 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_i8x32<const SHIFT: usize>(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i8x32<Avx512>,
b: i8x32<Avx512>,
shift: usize,
) -> i8x32<Avx512> {
if shift >= 32usize {
return b;
}
let idx = _mm256_add_epi8(
_mm256_setr_epi8(
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
),
_mm256_set1_epi8((shift) as i8),
);
let result = _mm256_permutex2var_epi8(
token.cvt_to_bytes_i8x32(a).val.0,
idx,
token.cvt_to_bytes_i8x32(b).val.0,
);
token.cvt_from_bytes_i8x32(u8x32 {
val: crate::support::Aligned256(result),
simd: token,
})
}
);
kernel(self, a, b, SHIFT)
}
#[inline(always)]
fn slide_within_blocks_i8x32<const SHIFT: usize>(
self,
a: i8x32<Self>,
b: i8x32<Self>,
) -> i8x32<Self> {
if SHIFT == 0 {
return a;
}
if SHIFT >= 16usize {
return b;
}
let a = self.cvt_to_bytes_i8x32(a).val.0;
let b = self.cvt_to_bytes_i8x32(b).val.0;
let result = dyn_alignr_256(self, b, a, SHIFT);
self.cvt_from_bytes_i8x32(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_within_blocks_i8x32(self, a: i8x32<Self>, indices: u8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, indices: u8x32<Avx512>) -> i8x32<Avx512> {
let bytes = token.cvt_to_bytes_i8x32(a).val.0;
let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into());
token.cvt_from_bytes_i8x32(u8x32 {
val: crate::support::Aligned256(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
_mm256_add_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
_mm256_sub_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
let dst_even = _mm256_mullo_epi16(a.into(), b.into());
let dst_odd = _mm256_mullo_epi16(
_mm256_srli_epi16::<8>(a.into()),
_mm256_srli_epi16::<8>(b.into()),
);
_mm256_or_si256(
_mm256_slli_epi16(dst_odd, 8),
_mm256_and_si256(dst_even, _mm256_set1_epi16(0xFF)),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
_mm256_and_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
_mm256_or_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
_mm256_xor_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i8x32(self, a: i8x32<Self>) -> i8x32<Self> {
a ^ !0
}
#[inline(always)]
fn shl_i8x32(self, a: i8x32<Self>, shift: u32) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, shift: u32) -> i8x32<Avx512> {
let val = a.into();
let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
let lo_16 =
_mm256_unpacklo_epi8(val, _mm256_cmpgt_epi8(_mm256_setzero_si256(), val));
let hi_16 =
_mm256_unpackhi_epi8(val, _mm256_cmpgt_epi8(_mm256_setzero_si256(), val));
let lo_shifted = _mm256_sll_epi16(lo_16, shift_count);
let hi_shifted = _mm256_sll_epi16(hi_16, shift_count);
_mm256_packs_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
let val = a.into();
let counts = b.into();
let zero = _mm256_setzero_si256();
let value_extend = zero;
let lo_values = _mm256_unpacklo_epi8(val, value_extend);
let hi_values = _mm256_unpackhi_epi8(val, value_extend);
let lo_counts = _mm256_unpacklo_epi8(counts, zero);
let hi_counts = _mm256_unpackhi_epi8(counts, zero);
let byte_mask = _mm256_set1_epi16(0x00ff);
let lo_shifted =
_mm256_and_si256(_mm256_sllv_epi16(lo_values, lo_counts), byte_mask);
let hi_shifted =
_mm256_and_si256(_mm256_sllv_epi16(hi_values, hi_counts), byte_mask);
_mm256_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_i8x32(self, a: i8x32<Self>, shift: u32) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, shift: u32) -> i8x32<Avx512> {
let val = a.into();
let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
let lo_16 =
_mm256_unpacklo_epi8(val, _mm256_cmpgt_epi8(_mm256_setzero_si256(), val));
let hi_16 =
_mm256_unpackhi_epi8(val, _mm256_cmpgt_epi8(_mm256_setzero_si256(), val));
let lo_shifted = _mm256_sra_epi16(lo_16, shift_count);
let hi_shifted = _mm256_sra_epi16(hi_16, shift_count);
_mm256_packs_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
let val = a.into();
let counts = b.into();
let zero = _mm256_setzero_si256();
let value_extend = _mm256_cmpgt_epi8(zero, val);
let lo_values = _mm256_unpacklo_epi8(val, value_extend);
let hi_values = _mm256_unpackhi_epi8(val, value_extend);
let lo_counts = _mm256_unpacklo_epi8(counts, zero);
let hi_counts = _mm256_unpackhi_epi8(counts, zero);
let byte_mask = _mm256_set1_epi16(0x00ff);
let lo_shifted =
_mm256_and_si256(_mm256_srav_epi16(lo_values, lo_counts), byte_mask);
let hi_shifted =
_mm256_and_si256(_mm256_srav_epi16(hi_values, hi_counts), byte_mask);
_mm256_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> mask8x32<Avx512> {
mask8x32 {
val: _mm256_cmpeq_epi8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> mask8x32<Avx512> {
mask8x32 {
val: _mm256_cmplt_epi8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> mask8x32<Avx512> {
mask8x32 {
val: _mm256_cmple_epi8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> mask8x32<Avx512> {
mask8x32 {
val: _mm256_cmpge_epi8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> mask8x32<Avx512> {
mask8x32 {
val: _mm256_cmpgt_epi8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
_mm256_permutex2var_epi8(
a.into(),
_mm256_setr_epi8(
0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, 8, 40, 9, 41, 10,
42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
_mm256_permutex2var_epi8(
a.into(),
_mm256_setr_epi8(
16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, 24, 56, 25,
57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
_mm256_permutex2var_epi8(
a.into(),
_mm256_setr_epi8(
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38,
40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
_mm256_permutex2var_epi8(
a.into(),
_mm256_setr_epi8(
1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39,
41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> (i8x32<Self>, i8x32<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i8x32<Avx512>,
b: i8x32<Avx512>,
) -> (i8x32<Avx512>, i8x32<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm256_permutex2var_epi8(
a,
_mm256_setr_epi8(
0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, 8, 40, 9, 41,
10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47,
),
b,
)
.simd_into(token),
_mm256_permutex2var_epi8(
a,
_mm256_setr_epi8(
16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, 24, 56,
25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> (i8x32<Self>, i8x32<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i8x32<Avx512>,
b: i8x32<Avx512>,
) -> (i8x32<Avx512>, i8x32<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm256_permutex2var_epi8(
a,
_mm256_setr_epi8(
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36,
38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62,
),
b,
)
.simd_into(token),
_mm256_permutex2var_epi8(
a,
_mm256_setr_epi8(
1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37,
39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_i8x32(self, a: mask8x32<Self>, b: i8x32<Self>, c: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask8x32<Avx512>,
b: i8x32<Avx512>,
c: i8x32<Avx512>,
) -> i8x32<Avx512> {
_mm256_mask_blend_epi8(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn min_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
_mm256_min_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x32<Avx512> {
_mm256_max_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn combine_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>, b: i8x32<Avx512>) -> i8x64<Avx512> {
_mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn split_i8x32(self, a: i8x32<Self>) -> (i8x16<Self>, i8x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>) -> (i8x16<Avx512>, i8x16<Avx512>) {
(
_mm256_extracti128_si256::<0>(a.into()).simd_into(token),
_mm256_extracti128_si256::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_i8x32(self, a: i8x32<Self>) -> i8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>) -> i8x32<Avx512> {
_mm256_sub_epi8(_mm256_setzero_si256(), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u8_i8x32(self, a: i8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>) -> u8x32<Avx512> {
__m256i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u32_i8x32(self, a: i8x32<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x32<Avx512>) -> u32x8<Avx512> {
__m256i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_u8x32(self, val: u8) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: u8) -> u8x32<Avx512> {
_mm256_set1_epi8(val.cast_signed()).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_u8x32(self, val: [u8; 32usize]) -> u8x32<Self> {
u8x32 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_u8x32(self, val: &[u8; 32usize]) -> u8x32<Self> {
u8x32 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_u8x32(self, a: u8x32<Self>) -> [u8; 32usize] {
crate::transmute::checked_transmute_copy::<__m256i, [u8; 32usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_u8x32(self, a: &u8x32<Self>) -> &[u8; 32usize] {
crate::transmute::checked_cast_ref::<__m256i, [u8; 32usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_u8x32(self, a: &mut u8x32<Self>) -> &mut [u8; 32usize] {
crate::transmute::checked_cast_mut::<__m256i, [u8; 32usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_u8x32(self, a: u8x32<Self>, dest: &mut [u8; 32usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_u8x32(self, a: u8x32<Self>) -> u8x32<Self> {
u8x32 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_u8x32(self, a: u8x32<Self>) -> u8x32<Self> {
u8x32 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_u8x32<const SHIFT: usize>(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u8x32<Avx512>,
b: u8x32<Avx512>,
shift: usize,
) -> u8x32<Avx512> {
if shift >= 32usize {
return b;
}
let idx = _mm256_add_epi8(
_mm256_setr_epi8(
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
),
_mm256_set1_epi8((shift) as i8),
);
let result = _mm256_permutex2var_epi8(
token.cvt_to_bytes_u8x32(a).val.0,
idx,
token.cvt_to_bytes_u8x32(b).val.0,
);
token.cvt_from_bytes_u8x32(u8x32 {
val: crate::support::Aligned256(result),
simd: token,
})
}
);
kernel(self, a, b, SHIFT)
}
#[inline(always)]
fn slide_within_blocks_u8x32<const SHIFT: usize>(
self,
a: u8x32<Self>,
b: u8x32<Self>,
) -> u8x32<Self> {
if SHIFT == 0 {
return a;
}
if SHIFT >= 16usize {
return b;
}
let a = self.cvt_to_bytes_u8x32(a).val.0;
let b = self.cvt_to_bytes_u8x32(b).val.0;
let result = dyn_alignr_256(self, b, a, SHIFT);
self.cvt_from_bytes_u8x32(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, indices: u8x32<Avx512>) -> u8x32<Avx512> {
let bytes = token.cvt_to_bytes_u8x32(a).val.0;
let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into());
token.cvt_from_bytes_u8x32(u8x32 {
val: crate::support::Aligned256(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
_mm256_add_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
_mm256_sub_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
let dst_even = _mm256_mullo_epi16(a.into(), b.into());
let dst_odd = _mm256_mullo_epi16(
_mm256_srli_epi16::<8>(a.into()),
_mm256_srli_epi16::<8>(b.into()),
);
_mm256_or_si256(
_mm256_slli_epi16(dst_odd, 8),
_mm256_and_si256(dst_even, _mm256_set1_epi16(0xFF)),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
_mm256_and_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
_mm256_or_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
_mm256_xor_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u8x32(self, a: u8x32<Self>) -> u8x32<Self> {
a ^ !0
}
#[inline(always)]
fn shl_u8x32(self, a: u8x32<Self>, shift: u32) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, shift: u32) -> u8x32<Avx512> {
let val = a.into();
let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
let lo_16 = _mm256_unpacklo_epi8(val, _mm256_setzero_si256());
let hi_16 = _mm256_unpackhi_epi8(val, _mm256_setzero_si256());
let lo_shifted = _mm256_sll_epi16(lo_16, shift_count);
let hi_shifted = _mm256_sll_epi16(hi_16, shift_count);
_mm256_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
let val = a.into();
let counts = b.into();
let zero = _mm256_setzero_si256();
let value_extend = zero;
let lo_values = _mm256_unpacklo_epi8(val, value_extend);
let hi_values = _mm256_unpackhi_epi8(val, value_extend);
let lo_counts = _mm256_unpacklo_epi8(counts, zero);
let hi_counts = _mm256_unpackhi_epi8(counts, zero);
let byte_mask = _mm256_set1_epi16(0x00ff);
let lo_shifted =
_mm256_and_si256(_mm256_sllv_epi16(lo_values, lo_counts), byte_mask);
let hi_shifted =
_mm256_and_si256(_mm256_sllv_epi16(hi_values, hi_counts), byte_mask);
_mm256_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_u8x32(self, a: u8x32<Self>, shift: u32) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, shift: u32) -> u8x32<Avx512> {
let val = a.into();
let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
let lo_16 = _mm256_unpacklo_epi8(val, _mm256_setzero_si256());
let hi_16 = _mm256_unpackhi_epi8(val, _mm256_setzero_si256());
let lo_shifted = _mm256_srl_epi16(lo_16, shift_count);
let hi_shifted = _mm256_srl_epi16(hi_16, shift_count);
_mm256_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
let val = a.into();
let counts = b.into();
let zero = _mm256_setzero_si256();
let value_extend = zero;
let lo_values = _mm256_unpacklo_epi8(val, value_extend);
let hi_values = _mm256_unpackhi_epi8(val, value_extend);
let lo_counts = _mm256_unpacklo_epi8(counts, zero);
let hi_counts = _mm256_unpackhi_epi8(counts, zero);
let byte_mask = _mm256_set1_epi16(0x00ff);
let lo_shifted =
_mm256_and_si256(_mm256_srlv_epi16(lo_values, lo_counts), byte_mask);
let hi_shifted =
_mm256_and_si256(_mm256_srlv_epi16(hi_values, hi_counts), byte_mask);
_mm256_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> mask8x32<Avx512> {
mask8x32 {
val: _mm256_cmpeq_epu8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> mask8x32<Avx512> {
mask8x32 {
val: _mm256_cmplt_epu8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> mask8x32<Avx512> {
mask8x32 {
val: _mm256_cmple_epu8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> mask8x32<Avx512> {
mask8x32 {
val: _mm256_cmpge_epu8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> mask8x32<Avx512> {
mask8x32 {
val: _mm256_cmpgt_epu8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
_mm256_permutex2var_epi8(
a.into(),
_mm256_setr_epi8(
0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, 8, 40, 9, 41, 10,
42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
_mm256_permutex2var_epi8(
a.into(),
_mm256_setr_epi8(
16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, 24, 56, 25,
57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
_mm256_permutex2var_epi8(
a.into(),
_mm256_setr_epi8(
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38,
40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
_mm256_permutex2var_epi8(
a.into(),
_mm256_setr_epi8(
1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37, 39,
41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> (u8x32<Self>, u8x32<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u8x32<Avx512>,
b: u8x32<Avx512>,
) -> (u8x32<Avx512>, u8x32<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm256_permutex2var_epi8(
a,
_mm256_setr_epi8(
0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, 8, 40, 9, 41,
10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47,
),
b,
)
.simd_into(token),
_mm256_permutex2var_epi8(
a,
_mm256_setr_epi8(
16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, 24, 56,
25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> (u8x32<Self>, u8x32<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u8x32<Avx512>,
b: u8x32<Avx512>,
) -> (u8x32<Avx512>, u8x32<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm256_permutex2var_epi8(
a,
_mm256_setr_epi8(
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36,
38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62,
),
b,
)
.simd_into(token),
_mm256_permutex2var_epi8(
a,
_mm256_setr_epi8(
1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31, 33, 35, 37,
39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_u8x32(self, a: mask8x32<Self>, b: u8x32<Self>, c: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask8x32<Avx512>,
b: u8x32<Avx512>,
c: u8x32<Avx512>,
) -> u8x32<Avx512> {
_mm256_mask_blend_epi8(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn min_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
_mm256_min_epu8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x32<Avx512> {
_mm256_max_epu8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn combine_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>, b: u8x32<Avx512>) -> u8x64<Avx512> {
_mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn split_u8x32(self, a: u8x32<Self>) -> (u8x16<Self>, u8x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>) -> (u8x16<Avx512>, u8x16<Avx512>) {
(
_mm256_extracti128_si256::<0>(a.into()).simd_into(token),
_mm256_extracti128_si256::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn widen_u8x32(self, a: u8x32<Self>) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>) -> u16x32<Avx512> {
_mm512_cvtepu8_epi16(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u32_u8x32(self, a: u8x32<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x32<Avx512>) -> u32x8<Avx512> {
__m256i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_mask8x32(self, val: bool) -> mask8x32<Self> {
mask8x32 {
val: (if val { 4294967295u64 } else { 0 }) as _,
simd: self,
}
}
#[inline(always)]
fn load_array_mask8x32(self, val: [i8; 32usize]) -> mask8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: [i8; 32usize]) -> mask8x32<Avx512> {
let lanes = crate::transmute::checked_transmute_copy(&val);
mask8x32 {
val: _mm256_movepi8_mask(lanes),
simd: token,
}
}
);
kernel(self, val)
}
#[inline(always)]
fn as_array_mask8x32(self, a: mask8x32<Self>) -> [i8; 32usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: mask8x32<Avx512>) -> [i8; 32usize] {
let lanes = _mm256_movm_epi8(a.val);
crate::transmute::checked_transmute_copy(&lanes)
}
);
kernel(self, a)
}
#[inline(always)]
fn from_bitmask_mask8x32(self, bits: u64) -> mask8x32<Self> {
mask8x32 {
val: (bits & 4294967295u64) as _,
simd: self,
}
}
#[inline(always)]
fn to_bitmask_mask8x32(self, a: mask8x32<Self>) -> u64 {
u64::from((a).val) & 4294967295u64
}
#[inline(always)]
fn set_mask8x32(self, a: &mut mask8x32<Self>, index: usize, value: bool) -> () {
assert!(
index < 32usize,
"mask lane index {index} is out of bounds for {} lanes",
32usize
);
let bit = 1u64 << index;
let bits = u64::from((a).val);
let bits = if value { bits | bit } else { bits & !bit };
*a = mask8x32 {
val: (bits) as _,
simd: self,
};
}
#[inline(always)]
fn and_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
mask8x32 {
val: ((u64::from((a).val) & u64::from((b).val)) & 4294967295u64) as _,
simd: self,
}
}
#[inline(always)]
fn or_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
mask8x32 {
val: ((u64::from((a).val) | u64::from((b).val)) & 4294967295u64) as _,
simd: self,
}
}
#[inline(always)]
fn xor_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
mask8x32 {
val: ((u64::from((a).val) ^ u64::from((b).val)) & 4294967295u64) as _,
simd: self,
}
}
#[inline(always)]
fn not_mask8x32(self, a: mask8x32<Self>) -> mask8x32<Self> {
mask8x32 {
val: ((!u64::from((a).val)) & 4294967295u64) as _,
simd: self,
}
}
#[inline(always)]
fn select_mask8x32(
self,
a: mask8x32<Self>,
b: mask8x32<Self>,
c: mask8x32<Self>,
) -> mask8x32<Self> {
mask8x32 {
val: (((u64::from((a).val) & u64::from((b).val))
| ((!u64::from((a).val)) & u64::from((c).val)))
& 4294967295u64) as _,
simd: self,
}
}
#[inline(always)]
fn simd_eq_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x32<Self> {
mask8x32 {
val: (!u64::from(a.val ^ b.val) & 4294967295u64) as _,
simd: self,
}
}
#[inline(always)]
fn any_true_mask8x32(self, a: mask8x32<Self>) -> bool {
let bits = u64::from((a).val) & 4294967295u64;
bits != 0
}
#[inline(always)]
fn all_true_mask8x32(self, a: mask8x32<Self>) -> bool {
let bits = u64::from((a).val) & 4294967295u64;
bits == 4294967295u64
}
#[inline(always)]
fn any_false_mask8x32(self, a: mask8x32<Self>) -> bool {
let bits = u64::from((a).val) & 4294967295u64;
bits != 4294967295u64
}
#[inline(always)]
fn all_false_mask8x32(self, a: mask8x32<Self>) -> bool {
let bits = u64::from((a).val) & 4294967295u64;
bits == 0
}
#[inline(always)]
fn combine_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x64<Self> {
let bits = (u64::from(a.val) | (u64::from(b.val) << 32usize)) & u64::MAX;
mask8x64 {
val: bits,
simd: self,
}
}
#[inline(always)]
fn split_mask8x32(self, a: mask8x32<Self>) -> (mask8x16<Self>, mask8x16<Self>) {
let bits = u64::from(a.val);
(
mask8x16 {
val: (bits & 65535u64) as _,
simd: self,
},
mask8x16 {
val: ((bits >> 16usize) & 65535u64) as _,
simd: self,
},
)
}
#[inline(always)]
fn splat_i16x16(self, val: i16) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: i16) -> i16x16<Avx512> {
_mm256_set1_epi16(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_i16x16(self, val: [i16; 16usize]) -> i16x16<Self> {
i16x16 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_i16x16(self, val: &[i16; 16usize]) -> i16x16<Self> {
i16x16 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_i16x16(self, a: i16x16<Self>) -> [i16; 16usize] {
crate::transmute::checked_transmute_copy::<__m256i, [i16; 16usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_i16x16(self, a: &i16x16<Self>) -> &[i16; 16usize] {
crate::transmute::checked_cast_ref::<__m256i, [i16; 16usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_i16x16(self, a: &mut i16x16<Self>) -> &mut [i16; 16usize] {
crate::transmute::checked_cast_mut::<__m256i, [i16; 16usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_i16x16(self, a: i16x16<Self>, dest: &mut [i16; 16usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_i16x16(self, a: u8x32<Self>) -> i16x16<Self> {
i16x16 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_i16x16(self, a: i16x16<Self>) -> u8x32<Self> {
u8x32 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_i16x16<const SHIFT: usize>(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i16x16<Avx512>,
b: i16x16<Avx512>,
shift: usize,
) -> i16x16<Avx512> {
if shift >= 16usize {
return b;
}
let idx = _mm256_add_epi8(
_mm256_setr_epi8(
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
),
_mm256_set1_epi8((shift * 2usize) as i8),
);
let result = _mm256_permutex2var_epi8(
token.cvt_to_bytes_i16x16(a).val.0,
idx,
token.cvt_to_bytes_i16x16(b).val.0,
);
token.cvt_from_bytes_i16x16(u8x32 {
val: crate::support::Aligned256(result),
simd: token,
})
}
);
kernel(self, a, b, SHIFT)
}
#[inline(always)]
fn slide_within_blocks_i16x16<const SHIFT: usize>(
self,
a: i16x16<Self>,
b: i16x16<Self>,
) -> i16x16<Self> {
if SHIFT == 0 {
return a;
}
if SHIFT >= 8usize {
return b;
}
let a = self.cvt_to_bytes_i16x16(a).val.0;
let b = self.cvt_to_bytes_i16x16(b).val.0;
let result = dyn_alignr_256(self, b, a, SHIFT * 2usize);
self.cvt_from_bytes_i16x16(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_within_blocks_i16x16(
self,
a: i16x16<Self>,
indices: u8x32<Self>,
) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, indices: u8x32<Avx512>) -> i16x16<Avx512> {
let bytes = token.cvt_to_bytes_i16x16(a).val.0;
let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into());
token.cvt_from_bytes_i16x16(u8x32 {
val: crate::support::Aligned256(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
_mm256_add_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
_mm256_sub_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
_mm256_mullo_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
_mm256_and_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
_mm256_or_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
_mm256_xor_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i16x16(self, a: i16x16<Self>) -> i16x16<Self> {
a ^ !0
}
#[inline(always)]
fn shl_i16x16(self, a: i16x16<Self>, shift: u32) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, shift: u32) -> i16x16<Avx512> {
_mm256_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
_mm256_sllv_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_i16x16(self, a: i16x16<Self>, shift: u32) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, shift: u32) -> i16x16<Avx512> {
_mm256_sra_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
_mm256_srav_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> mask16x16<Avx512> {
mask16x16 {
val: _mm256_cmpeq_epi16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> mask16x16<Avx512> {
mask16x16 {
val: _mm256_cmplt_epi16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> mask16x16<Avx512> {
mask16x16 {
val: _mm256_cmple_epi16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> mask16x16<Avx512> {
mask16x16 {
val: _mm256_cmpge_epi16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> mask16x16<Avx512> {
mask16x16 {
val: _mm256_cmpgt_epi16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
_mm256_permutex2var_epi16(
a.into(),
_mm256_setr_epi16(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
_mm256_permutex2var_epi16(
a.into(),
_mm256_setr_epi16(8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
_mm256_permutex2var_epi16(
a.into(),
_mm256_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
_mm256_permutex2var_epi16(
a.into(),
_mm256_setr_epi16(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> (i16x16<Self>, i16x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i16x16<Avx512>,
b: i16x16<Avx512>,
) -> (i16x16<Avx512>, i16x16<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm256_permutex2var_epi16(
a,
_mm256_setr_epi16(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23),
b,
)
.simd_into(token),
_mm256_permutex2var_epi16(
a,
_mm256_setr_epi16(
8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> (i16x16<Self>, i16x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i16x16<Avx512>,
b: i16x16<Avx512>,
) -> (i16x16<Avx512>, i16x16<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm256_permutex2var_epi16(
a,
_mm256_setr_epi16(
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30,
),
b,
)
.simd_into(token),
_mm256_permutex2var_epi16(
a,
_mm256_setr_epi16(
1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_i16x16(self, a: mask16x16<Self>, b: i16x16<Self>, c: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask16x16<Avx512>,
b: i16x16<Avx512>,
c: i16x16<Avx512>,
) -> i16x16<Avx512> {
_mm256_mask_blend_epi16(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn min_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
_mm256_min_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x16<Avx512> {
_mm256_max_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn combine_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>, b: i16x16<Avx512>) -> i16x32<Avx512> {
_mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn split_i16x16(self, a: i16x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>) -> (i16x8<Avx512>, i16x8<Avx512>) {
(
_mm256_extracti128_si256::<0>(a.into()).simd_into(token),
_mm256_extracti128_si256::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_i16x16(self, a: i16x16<Self>) -> i16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>) -> i16x16<Avx512> {
_mm256_sub_epi16(_mm256_setzero_si256(), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u8_i16x16(self, a: i16x16<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>) -> u8x32<Avx512> {
__m256i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u32_i16x16(self, a: i16x16<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x16<Avx512>) -> u32x8<Avx512> {
__m256i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_u16x16(self, val: u16) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: u16) -> u16x16<Avx512> {
_mm256_set1_epi16(val.cast_signed()).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_u16x16(self, val: [u16; 16usize]) -> u16x16<Self> {
u16x16 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_u16x16(self, val: &[u16; 16usize]) -> u16x16<Self> {
u16x16 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_u16x16(self, a: u16x16<Self>) -> [u16; 16usize] {
crate::transmute::checked_transmute_copy::<__m256i, [u16; 16usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_u16x16(self, a: &u16x16<Self>) -> &[u16; 16usize] {
crate::transmute::checked_cast_ref::<__m256i, [u16; 16usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_u16x16(self, a: &mut u16x16<Self>) -> &mut [u16; 16usize] {
crate::transmute::checked_cast_mut::<__m256i, [u16; 16usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_u16x16(self, a: u16x16<Self>, dest: &mut [u16; 16usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_u16x16(self, a: u8x32<Self>) -> u16x16<Self> {
u16x16 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_u16x16(self, a: u16x16<Self>) -> u8x32<Self> {
u8x32 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_u16x16<const SHIFT: usize>(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u16x16<Avx512>,
b: u16x16<Avx512>,
shift: usize,
) -> u16x16<Avx512> {
if shift >= 16usize {
return b;
}
let idx = _mm256_add_epi8(
_mm256_setr_epi8(
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
),
_mm256_set1_epi8((shift * 2usize) as i8),
);
let result = _mm256_permutex2var_epi8(
token.cvt_to_bytes_u16x16(a).val.0,
idx,
token.cvt_to_bytes_u16x16(b).val.0,
);
token.cvt_from_bytes_u16x16(u8x32 {
val: crate::support::Aligned256(result),
simd: token,
})
}
);
kernel(self, a, b, SHIFT)
}
#[inline(always)]
fn slide_within_blocks_u16x16<const SHIFT: usize>(
self,
a: u16x16<Self>,
b: u16x16<Self>,
) -> u16x16<Self> {
if SHIFT == 0 {
return a;
}
if SHIFT >= 8usize {
return b;
}
let a = self.cvt_to_bytes_u16x16(a).val.0;
let b = self.cvt_to_bytes_u16x16(b).val.0;
let result = dyn_alignr_256(self, b, a, SHIFT * 2usize);
self.cvt_from_bytes_u16x16(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_within_blocks_u16x16(
self,
a: u16x16<Self>,
indices: u8x32<Self>,
) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, indices: u8x32<Avx512>) -> u16x16<Avx512> {
let bytes = token.cvt_to_bytes_u16x16(a).val.0;
let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into());
token.cvt_from_bytes_u16x16(u8x32 {
val: crate::support::Aligned256(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
_mm256_add_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
_mm256_sub_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
_mm256_mullo_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
_mm256_and_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
_mm256_or_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
_mm256_xor_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u16x16(self, a: u16x16<Self>) -> u16x16<Self> {
a ^ !0
}
#[inline(always)]
fn shl_u16x16(self, a: u16x16<Self>, shift: u32) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, shift: u32) -> u16x16<Avx512> {
_mm256_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
_mm256_sllv_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_u16x16(self, a: u16x16<Self>, shift: u32) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, shift: u32) -> u16x16<Avx512> {
_mm256_srl_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
_mm256_srlv_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> mask16x16<Avx512> {
mask16x16 {
val: _mm256_cmpeq_epu16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> mask16x16<Avx512> {
mask16x16 {
val: _mm256_cmplt_epu16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> mask16x16<Avx512> {
mask16x16 {
val: _mm256_cmple_epu16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> mask16x16<Avx512> {
mask16x16 {
val: _mm256_cmpge_epu16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> mask16x16<Avx512> {
mask16x16 {
val: _mm256_cmpgt_epu16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
_mm256_permutex2var_epi16(
a.into(),
_mm256_setr_epi16(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
_mm256_permutex2var_epi16(
a.into(),
_mm256_setr_epi16(8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
_mm256_permutex2var_epi16(
a.into(),
_mm256_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
_mm256_permutex2var_epi16(
a.into(),
_mm256_setr_epi16(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> (u16x16<Self>, u16x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u16x16<Avx512>,
b: u16x16<Avx512>,
) -> (u16x16<Avx512>, u16x16<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm256_permutex2var_epi16(
a,
_mm256_setr_epi16(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23),
b,
)
.simd_into(token),
_mm256_permutex2var_epi16(
a,
_mm256_setr_epi16(
8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> (u16x16<Self>, u16x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u16x16<Avx512>,
b: u16x16<Avx512>,
) -> (u16x16<Avx512>, u16x16<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm256_permutex2var_epi16(
a,
_mm256_setr_epi16(
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30,
),
b,
)
.simd_into(token),
_mm256_permutex2var_epi16(
a,
_mm256_setr_epi16(
1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_u16x16(self, a: mask16x16<Self>, b: u16x16<Self>, c: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask16x16<Avx512>,
b: u16x16<Avx512>,
c: u16x16<Avx512>,
) -> u16x16<Avx512> {
_mm256_mask_blend_epi16(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn min_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
_mm256_min_epu16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x16<Avx512> {
_mm256_max_epu16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn combine_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>, b: u16x16<Avx512>) -> u16x32<Avx512> {
_mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn split_u16x16(self, a: u16x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>) -> (u16x8<Avx512>, u16x8<Avx512>) {
(
_mm256_extracti128_si256::<0>(a.into()).simd_into(token),
_mm256_extracti128_si256::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn narrow_u16x16(self, a: u16x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>) -> u8x16<Avx512> {
_mm256_cvtepi16_epi8(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u8_u16x16(self, a: u16x16<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>) -> u8x32<Avx512> {
__m256i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u32_u16x16(self, a: u16x16<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x16<Avx512>) -> u32x8<Avx512> {
__m256i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_mask16x16(self, val: bool) -> mask16x16<Self> {
mask16x16 {
val: (if val { 65535u64 } else { 0 }) as _,
simd: self,
}
}
#[inline(always)]
fn load_array_mask16x16(self, val: [i16; 16usize]) -> mask16x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: [i16; 16usize]) -> mask16x16<Avx512> {
let lanes = crate::transmute::checked_transmute_copy(&val);
mask16x16 {
val: _mm256_movepi16_mask(lanes),
simd: token,
}
}
);
kernel(self, val)
}
#[inline(always)]
fn as_array_mask16x16(self, a: mask16x16<Self>) -> [i16; 16usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: mask16x16<Avx512>) -> [i16; 16usize] {
let lanes = _mm256_movm_epi16(a.val);
crate::transmute::checked_transmute_copy(&lanes)
}
);
kernel(self, a)
}
#[inline(always)]
fn from_bitmask_mask16x16(self, bits: u64) -> mask16x16<Self> {
mask16x16 {
val: (bits & 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn to_bitmask_mask16x16(self, a: mask16x16<Self>) -> u64 {
u64::from((a).val) & 65535u64
}
#[inline(always)]
fn set_mask16x16(self, a: &mut mask16x16<Self>, index: usize, value: bool) -> () {
assert!(
index < 16usize,
"mask lane index {index} is out of bounds for {} lanes",
16usize
);
let bit = 1u64 << index;
let bits = u64::from((a).val);
let bits = if value { bits | bit } else { bits & !bit };
*a = mask16x16 {
val: (bits) as _,
simd: self,
};
}
#[inline(always)]
fn and_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
mask16x16 {
val: ((u64::from((a).val) & u64::from((b).val)) & 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn or_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
mask16x16 {
val: ((u64::from((a).val) | u64::from((b).val)) & 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn xor_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
mask16x16 {
val: ((u64::from((a).val) ^ u64::from((b).val)) & 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn not_mask16x16(self, a: mask16x16<Self>) -> mask16x16<Self> {
mask16x16 {
val: ((!u64::from((a).val)) & 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn select_mask16x16(
self,
a: mask16x16<Self>,
b: mask16x16<Self>,
c: mask16x16<Self>,
) -> mask16x16<Self> {
mask16x16 {
val: (((u64::from((a).val) & u64::from((b).val))
| ((!u64::from((a).val)) & u64::from((c).val)))
& 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn simd_eq_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x16<Self> {
mask16x16 {
val: (!u64::from(a.val ^ b.val) & 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn any_true_mask16x16(self, a: mask16x16<Self>) -> bool {
let bits = u64::from((a).val) & 65535u64;
bits != 0
}
#[inline(always)]
fn all_true_mask16x16(self, a: mask16x16<Self>) -> bool {
let bits = u64::from((a).val) & 65535u64;
bits == 65535u64
}
#[inline(always)]
fn any_false_mask16x16(self, a: mask16x16<Self>) -> bool {
let bits = u64::from((a).val) & 65535u64;
bits != 65535u64
}
#[inline(always)]
fn all_false_mask16x16(self, a: mask16x16<Self>) -> bool {
let bits = u64::from((a).val) & 65535u64;
bits == 0
}
#[inline(always)]
fn combine_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x32<Self> {
let bits = (u64::from(a.val) | (u64::from(b.val) << 16usize)) & 4294967295u64;
mask16x32 {
val: bits as _,
simd: self,
}
}
#[inline(always)]
fn split_mask16x16(self, a: mask16x16<Self>) -> (mask16x8<Self>, mask16x8<Self>) {
let bits = u64::from(a.val);
(
mask16x8 {
val: (bits & 255u64) as _,
simd: self,
},
mask16x8 {
val: ((bits >> 8usize) & 255u64) as _,
simd: self,
},
)
}
#[inline(always)]
fn splat_i32x8(self, val: i32) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: i32) -> i32x8<Avx512> {
_mm256_set1_epi32(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_i32x8(self, val: [i32; 8usize]) -> i32x8<Self> {
i32x8 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_i32x8(self, val: &[i32; 8usize]) -> i32x8<Self> {
i32x8 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_i32x8(self, a: i32x8<Self>) -> [i32; 8usize] {
crate::transmute::checked_transmute_copy::<__m256i, [i32; 8usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_i32x8(self, a: &i32x8<Self>) -> &[i32; 8usize] {
crate::transmute::checked_cast_ref::<__m256i, [i32; 8usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_i32x8(self, a: &mut i32x8<Self>) -> &mut [i32; 8usize] {
crate::transmute::checked_cast_mut::<__m256i, [i32; 8usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_i32x8(self, a: i32x8<Self>, dest: &mut [i32; 8usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_i32x8(self, a: u8x32<Self>) -> i32x8<Self> {
i32x8 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_i32x8(self, a: i32x8<Self>) -> u8x32<Self> {
u8x32 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_i32x8<const SHIFT: usize>(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i32x8<Avx512>,
b: i32x8<Avx512>,
shift: usize,
) -> i32x8<Avx512> {
if shift >= 8usize {
return b;
}
let idx = _mm256_add_epi8(
_mm256_setr_epi8(
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
),
_mm256_set1_epi8((shift * 4usize) as i8),
);
let result = _mm256_permutex2var_epi8(
token.cvt_to_bytes_i32x8(a).val.0,
idx,
token.cvt_to_bytes_i32x8(b).val.0,
);
token.cvt_from_bytes_i32x8(u8x32 {
val: crate::support::Aligned256(result),
simd: token,
})
}
);
kernel(self, a, b, SHIFT)
}
#[inline(always)]
fn slide_within_blocks_i32x8<const SHIFT: usize>(
self,
a: i32x8<Self>,
b: i32x8<Self>,
) -> i32x8<Self> {
if SHIFT == 0 {
return a;
}
if SHIFT >= 4usize {
return b;
}
let a = self.cvt_to_bytes_i32x8(a).val.0;
let b = self.cvt_to_bytes_i32x8(b).val.0;
let result = dyn_alignr_256(self, b, a, SHIFT * 4usize);
self.cvt_from_bytes_i32x8(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_within_blocks_i32x8(self, a: i32x8<Self>, indices: u8x32<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, indices: u8x32<Avx512>) -> i32x8<Avx512> {
let bytes = token.cvt_to_bytes_i32x8(a).val.0;
let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into());
token.cvt_from_bytes_i32x8(u8x32 {
val: crate::support::Aligned256(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
_mm256_add_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
_mm256_sub_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
_mm256_mullo_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
_mm256_and_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
_mm256_or_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
_mm256_xor_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i32x8(self, a: i32x8<Self>) -> i32x8<Self> {
a ^ !0
}
#[inline(always)]
fn shl_i32x8(self, a: i32x8<Self>, shift: u32) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, shift: u32) -> i32x8<Avx512> {
_mm256_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
_mm256_sllv_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_i32x8(self, a: i32x8<Self>, shift: u32) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, shift: u32) -> i32x8<Avx512> {
_mm256_sra_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
_mm256_srav_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> mask32x8<Avx512> {
mask32x8 {
val: _mm256_cmpeq_epi32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> mask32x8<Avx512> {
mask32x8 {
val: _mm256_cmplt_epi32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> mask32x8<Avx512> {
mask32x8 {
val: _mm256_cmple_epi32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> mask32x8<Avx512> {
mask32x8 {
val: _mm256_cmpge_epi32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> mask32x8<Avx512> {
mask32x8 {
val: _mm256_cmpgt_epi32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
_mm256_permutex2var_epi32(
a.into(),
_mm256_setr_epi32(0, 8, 1, 9, 2, 10, 3, 11),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
_mm256_permutex2var_epi32(
a.into(),
_mm256_setr_epi32(4, 12, 5, 13, 6, 14, 7, 15),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
_mm256_permutex2var_epi32(
a.into(),
_mm256_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
_mm256_permutex2var_epi32(
a.into(),
_mm256_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> (i32x8<Self>, i32x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i32x8<Avx512>,
b: i32x8<Avx512>,
) -> (i32x8<Avx512>, i32x8<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm256_permutex2var_epi32(a, _mm256_setr_epi32(0, 8, 1, 9, 2, 10, 3, 11), b)
.simd_into(token),
_mm256_permutex2var_epi32(a, _mm256_setr_epi32(4, 12, 5, 13, 6, 14, 7, 15), b)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> (i32x8<Self>, i32x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i32x8<Avx512>,
b: i32x8<Avx512>,
) -> (i32x8<Avx512>, i32x8<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm256_permutex2var_epi32(a, _mm256_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14), b)
.simd_into(token),
_mm256_permutex2var_epi32(a, _mm256_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15), b)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_i32x8(self, a: mask32x8<Self>, b: i32x8<Self>, c: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask32x8<Avx512>,
b: i32x8<Avx512>,
c: i32x8<Avx512>,
) -> i32x8<Avx512> {
_mm256_mask_blend_epi32(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn min_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
_mm256_min_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x8<Avx512> {
_mm256_max_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn combine_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>, b: i32x8<Avx512>) -> i32x16<Avx512> {
_mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn split_i32x8(self, a: i32x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>) -> (i32x4<Avx512>, i32x4<Avx512>) {
(
_mm256_extracti128_si256::<0>(a.into()).simd_into(token),
_mm256_extracti128_si256::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_i32x8(self, a: i32x8<Self>) -> i32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>) -> i32x8<Avx512> {
_mm256_sub_epi32(_mm256_setzero_si256(), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u8_i32x8(self, a: i32x8<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>) -> u8x32<Avx512> {
__m256i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u32_i32x8(self, a: i32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>) -> u32x8<Avx512> {
__m256i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_f32_i32x8(self, a: i32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x8<Avx512>) -> f32x8<Avx512> {
_mm256_cvtepi32_ps(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_u32x8(self, val: u32) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: u32) -> u32x8<Avx512> {
_mm256_set1_epi32(val.cast_signed()).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_u32x8(self, val: [u32; 8usize]) -> u32x8<Self> {
u32x8 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_u32x8(self, val: &[u32; 8usize]) -> u32x8<Self> {
u32x8 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_u32x8(self, a: u32x8<Self>) -> [u32; 8usize] {
crate::transmute::checked_transmute_copy::<__m256i, [u32; 8usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_u32x8(self, a: &u32x8<Self>) -> &[u32; 8usize] {
crate::transmute::checked_cast_ref::<__m256i, [u32; 8usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_u32x8(self, a: &mut u32x8<Self>) -> &mut [u32; 8usize] {
crate::transmute::checked_cast_mut::<__m256i, [u32; 8usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_u32x8(self, a: u32x8<Self>, dest: &mut [u32; 8usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_u32x8(self, a: u8x32<Self>) -> u32x8<Self> {
u32x8 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_u32x8(self, a: u32x8<Self>) -> u8x32<Self> {
u8x32 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_u32x8<const SHIFT: usize>(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u32x8<Avx512>,
b: u32x8<Avx512>,
shift: usize,
) -> u32x8<Avx512> {
if shift >= 8usize {
return b;
}
let idx = _mm256_add_epi8(
_mm256_setr_epi8(
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
),
_mm256_set1_epi8((shift * 4usize) as i8),
);
let result = _mm256_permutex2var_epi8(
token.cvt_to_bytes_u32x8(a).val.0,
idx,
token.cvt_to_bytes_u32x8(b).val.0,
);
token.cvt_from_bytes_u32x8(u8x32 {
val: crate::support::Aligned256(result),
simd: token,
})
}
);
kernel(self, a, b, SHIFT)
}
#[inline(always)]
fn slide_within_blocks_u32x8<const SHIFT: usize>(
self,
a: u32x8<Self>,
b: u32x8<Self>,
) -> u32x8<Self> {
if SHIFT == 0 {
return a;
}
if SHIFT >= 4usize {
return b;
}
let a = self.cvt_to_bytes_u32x8(a).val.0;
let b = self.cvt_to_bytes_u32x8(b).val.0;
let result = dyn_alignr_256(self, b, a, SHIFT * 4usize);
self.cvt_from_bytes_u32x8(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_within_blocks_u32x8(self, a: u32x8<Self>, indices: u8x32<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, indices: u8x32<Avx512>) -> u32x8<Avx512> {
let bytes = token.cvt_to_bytes_u32x8(a).val.0;
let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into());
token.cvt_from_bytes_u32x8(u8x32 {
val: crate::support::Aligned256(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
_mm256_add_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
_mm256_sub_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
_mm256_mullo_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
_mm256_and_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
_mm256_or_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
_mm256_xor_si256(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u32x8(self, a: u32x8<Self>) -> u32x8<Self> {
a ^ !0
}
#[inline(always)]
fn shl_u32x8(self, a: u32x8<Self>, shift: u32) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, shift: u32) -> u32x8<Avx512> {
_mm256_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
_mm256_sllv_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_u32x8(self, a: u32x8<Self>, shift: u32) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, shift: u32) -> u32x8<Avx512> {
_mm256_srl_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
_mm256_srlv_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> mask32x8<Avx512> {
mask32x8 {
val: _mm256_cmpeq_epu32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> mask32x8<Avx512> {
mask32x8 {
val: _mm256_cmplt_epu32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> mask32x8<Avx512> {
mask32x8 {
val: _mm256_cmple_epu32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> mask32x8<Avx512> {
mask32x8 {
val: _mm256_cmpge_epu32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> mask32x8<Avx512> {
mask32x8 {
val: _mm256_cmpgt_epu32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
_mm256_permutex2var_epi32(
a.into(),
_mm256_setr_epi32(0, 8, 1, 9, 2, 10, 3, 11),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
_mm256_permutex2var_epi32(
a.into(),
_mm256_setr_epi32(4, 12, 5, 13, 6, 14, 7, 15),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
_mm256_permutex2var_epi32(
a.into(),
_mm256_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
_mm256_permutex2var_epi32(
a.into(),
_mm256_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> (u32x8<Self>, u32x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u32x8<Avx512>,
b: u32x8<Avx512>,
) -> (u32x8<Avx512>, u32x8<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm256_permutex2var_epi32(a, _mm256_setr_epi32(0, 8, 1, 9, 2, 10, 3, 11), b)
.simd_into(token),
_mm256_permutex2var_epi32(a, _mm256_setr_epi32(4, 12, 5, 13, 6, 14, 7, 15), b)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> (u32x8<Self>, u32x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u32x8<Avx512>,
b: u32x8<Avx512>,
) -> (u32x8<Avx512>, u32x8<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm256_permutex2var_epi32(a, _mm256_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14), b)
.simd_into(token),
_mm256_permutex2var_epi32(a, _mm256_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15), b)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_u32x8(self, a: mask32x8<Self>, b: u32x8<Self>, c: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask32x8<Avx512>,
b: u32x8<Avx512>,
c: u32x8<Avx512>,
) -> u32x8<Avx512> {
_mm256_mask_blend_epi32(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn min_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
_mm256_min_epu32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x8<Avx512> {
_mm256_max_epu32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn combine_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>, b: u32x8<Avx512>) -> u32x16<Avx512> {
_mm512_inserti64x4::<1>(_mm512_castsi256_si512(a.into()), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn split_u32x8(self, a: u32x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>) -> (u32x4<Avx512>, u32x4<Avx512>) {
(
_mm256_extracti128_si256::<0>(a.into()).simd_into(token),
_mm256_extracti128_si256::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u8_u32x8(self, a: u32x8<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>) -> u8x32<Avx512> {
__m256i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_f32_u32x8(self, a: u32x8<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x8<Avx512>) -> f32x8<Avx512> {
_mm512_castps512_ps256(_mm512_cvtepu32_ps(_mm512_zextsi256_si512(a.into())))
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_mask32x8(self, val: bool) -> mask32x8<Self> {
mask32x8 {
val: (if val { 255u64 } else { 0 }) as _,
simd: self,
}
}
#[inline(always)]
fn load_array_mask32x8(self, val: [i32; 8usize]) -> mask32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: [i32; 8usize]) -> mask32x8<Avx512> {
let lanes = crate::transmute::checked_transmute_copy(&val);
mask32x8 {
val: _mm256_movepi32_mask(lanes),
simd: token,
}
}
);
kernel(self, val)
}
#[inline(always)]
fn as_array_mask32x8(self, a: mask32x8<Self>) -> [i32; 8usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: mask32x8<Avx512>) -> [i32; 8usize] {
let lanes = _mm256_movm_epi32(a.val);
crate::transmute::checked_transmute_copy(&lanes)
}
);
kernel(self, a)
}
#[inline(always)]
fn from_bitmask_mask32x8(self, bits: u64) -> mask32x8<Self> {
mask32x8 {
val: (bits & 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn to_bitmask_mask32x8(self, a: mask32x8<Self>) -> u64 {
u64::from((a).val) & 255u64
}
#[inline(always)]
fn set_mask32x8(self, a: &mut mask32x8<Self>, index: usize, value: bool) -> () {
assert!(
index < 8usize,
"mask lane index {index} is out of bounds for {} lanes",
8usize
);
let bit = 1u64 << index;
let bits = u64::from((a).val);
let bits = if value { bits | bit } else { bits & !bit };
*a = mask32x8 {
val: (bits) as _,
simd: self,
};
}
#[inline(always)]
fn and_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
mask32x8 {
val: ((u64::from((a).val) & u64::from((b).val)) & 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn or_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
mask32x8 {
val: ((u64::from((a).val) | u64::from((b).val)) & 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn xor_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
mask32x8 {
val: ((u64::from((a).val) ^ u64::from((b).val)) & 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn not_mask32x8(self, a: mask32x8<Self>) -> mask32x8<Self> {
mask32x8 {
val: ((!u64::from((a).val)) & 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn select_mask32x8(
self,
a: mask32x8<Self>,
b: mask32x8<Self>,
c: mask32x8<Self>,
) -> mask32x8<Self> {
mask32x8 {
val: (((u64::from((a).val) & u64::from((b).val))
| ((!u64::from((a).val)) & u64::from((c).val)))
& 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn simd_eq_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x8<Self> {
mask32x8 {
val: (!u64::from(a.val ^ b.val) & 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn any_true_mask32x8(self, a: mask32x8<Self>) -> bool {
let bits = u64::from((a).val) & 255u64;
bits != 0
}
#[inline(always)]
fn all_true_mask32x8(self, a: mask32x8<Self>) -> bool {
let bits = u64::from((a).val) & 255u64;
bits == 255u64
}
#[inline(always)]
fn any_false_mask32x8(self, a: mask32x8<Self>) -> bool {
let bits = u64::from((a).val) & 255u64;
bits != 255u64
}
#[inline(always)]
fn all_false_mask32x8(self, a: mask32x8<Self>) -> bool {
let bits = u64::from((a).val) & 255u64;
bits == 0
}
#[inline(always)]
fn combine_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x16<Self> {
let bits = (u64::from(a.val) | (u64::from(b.val) << 8usize)) & 65535u64;
mask32x16 {
val: bits as _,
simd: self,
}
}
#[inline(always)]
fn split_mask32x8(self, a: mask32x8<Self>) -> (mask32x4<Self>, mask32x4<Self>) {
let bits = u64::from(a.val);
(
mask32x4 {
val: (bits & 15u64) as _,
simd: self,
},
mask32x4 {
val: ((bits >> 4usize) & 15u64) as _,
simd: self,
},
)
}
#[inline(always)]
fn splat_f64x4(self, val: f64) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: f64) -> f64x4<Avx512> {
_mm256_set1_pd(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_f64x4(self, val: [f64; 4usize]) -> f64x4<Self> {
f64x4 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_f64x4(self, val: &[f64; 4usize]) -> f64x4<Self> {
f64x4 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_f64x4(self, a: f64x4<Self>) -> [f64; 4usize] {
crate::transmute::checked_transmute_copy::<__m256d, [f64; 4usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_f64x4(self, a: &f64x4<Self>) -> &[f64; 4usize] {
crate::transmute::checked_cast_ref::<__m256d, [f64; 4usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_f64x4(self, a: &mut f64x4<Self>) -> &mut [f64; 4usize] {
crate::transmute::checked_cast_mut::<__m256d, [f64; 4usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_f64x4(self, a: f64x4<Self>, dest: &mut [f64; 4usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_f64x4(self, a: u8x32<Self>) -> f64x4<Self> {
f64x4 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_f64x4(self, a: f64x4<Self>) -> u8x32<Self> {
u8x32 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_f64x4<const SHIFT: usize>(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f64x4<Avx512>,
b: f64x4<Avx512>,
shift: usize,
) -> f64x4<Avx512> {
if shift >= 4usize {
return b;
}
let idx = _mm256_add_epi8(
_mm256_setr_epi8(
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
),
_mm256_set1_epi8((shift * 8usize) as i8),
);
let result = _mm256_permutex2var_epi8(
token.cvt_to_bytes_f64x4(a).val.0,
idx,
token.cvt_to_bytes_f64x4(b).val.0,
);
token.cvt_from_bytes_f64x4(u8x32 {
val: crate::support::Aligned256(result),
simd: token,
})
}
);
kernel(self, a, b, SHIFT)
}
#[inline(always)]
fn slide_within_blocks_f64x4<const SHIFT: usize>(
self,
a: f64x4<Self>,
b: f64x4<Self>,
) -> f64x4<Self> {
if SHIFT == 0 {
return a;
}
if SHIFT >= 2usize {
return b;
}
let a = self.cvt_to_bytes_f64x4(a).val.0;
let b = self.cvt_to_bytes_f64x4(b).val.0;
let result = dyn_alignr_256(self, b, a, SHIFT * 8usize);
self.cvt_from_bytes_f64x4(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_within_blocks_f64x4(self, a: f64x4<Self>, indices: u8x32<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>, indices: u8x32<Avx512>) -> f64x4<Avx512> {
let bytes = token.cvt_to_bytes_f64x4(a).val.0;
let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into());
token.cvt_from_bytes_f64x4(u8x32 {
val: crate::support::Aligned256(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn abs_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>) -> f64x4<Avx512> {
_mm256_andnot_pd(_mm256_set1_pd(-0.0), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>) -> f64x4<Avx512> {
_mm256_xor_pd(a.into(), _mm256_set1_pd(-0.0)).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn sqrt_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>) -> f64x4<Avx512> {
_mm256_sqrt_pd(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn approximate_recip_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>) -> f64x4<Avx512> {
_mm256_rcp14_pd(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn add_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
_mm256_add_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
_mm256_sub_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
_mm256_mul_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn div_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
_mm256_div_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn copysign_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
let mask = _mm256_set1_pd(-0.0);
_mm256_or_pd(
_mm256_and_pd(mask, b.into()),
_mm256_andnot_pd(mask, a.into()),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> mask64x4<Avx512> {
mask64x4 {
val: _mm256_cmp_pd_mask::<0i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> mask64x4<Avx512> {
mask64x4 {
val: _mm256_cmp_pd_mask::<17i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> mask64x4<Avx512> {
mask64x4 {
val: _mm256_cmp_pd_mask::<18i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> mask64x4<Avx512> {
mask64x4 {
val: _mm256_cmp_pd_mask::<29i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> mask64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> mask64x4<Avx512> {
mask64x4 {
val: _mm256_cmp_pd_mask::<30i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
_mm256_permutex2var_pd(a.into(), _mm256_setr_epi64x(0, 4, 1, 5), b.into())
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
_mm256_permutex2var_pd(a.into(), _mm256_setr_epi64x(2, 6, 3, 7), b.into())
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
_mm256_permutex2var_pd(a.into(), _mm256_setr_epi64x(0, 2, 4, 6), b.into())
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
_mm256_permutex2var_pd(a.into(), _mm256_setr_epi64x(1, 3, 5, 7), b.into())
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> (f64x4<Self>, f64x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f64x4<Avx512>,
b: f64x4<Avx512>,
) -> (f64x4<Avx512>, f64x4<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm256_permutex2var_pd(a, _mm256_setr_epi64x(0, 4, 1, 5), b).simd_into(token),
_mm256_permutex2var_pd(a, _mm256_setr_epi64x(2, 6, 3, 7), b).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> (f64x4<Self>, f64x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f64x4<Avx512>,
b: f64x4<Avx512>,
) -> (f64x4<Avx512>, f64x4<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm256_permutex2var_pd(a, _mm256_setr_epi64x(0, 2, 4, 6), b).simd_into(token),
_mm256_permutex2var_pd(a, _mm256_setr_epi64x(1, 3, 5, 7), b).simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
_mm256_max_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
_mm256_min_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_precise_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
_mm256_range_pd::<5i32>(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_precise_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x4<Avx512> {
_mm256_range_pd::<4i32>(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_add_f64x4(self, a: f64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f64x4<Avx512>,
b: f64x4<Avx512>,
c: f64x4<Avx512>,
) -> f64x4<Avx512> {
_mm256_fmadd_pd(a.into(), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn mul_sub_f64x4(self, a: f64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f64x4<Avx512>,
b: f64x4<Avx512>,
c: f64x4<Avx512>,
) -> f64x4<Avx512> {
_mm256_fmsub_pd(a.into(), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn floor_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>) -> f64x4<Avx512> {
_mm256_round_pd::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn ceil_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>) -> f64x4<Avx512> {
_mm256_round_pd::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn round_ties_even_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>) -> f64x4<Avx512> {
_mm256_round_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn fract_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
a - self.trunc_f64x4(a)
}
#[inline(always)]
fn trunc_f64x4(self, a: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>) -> f64x4<Avx512> {
_mm256_round_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn select_f64x4(self, a: mask64x4<Self>, b: f64x4<Self>, c: f64x4<Self>) -> f64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask64x4<Avx512>,
b: f64x4<Avx512>,
c: f64x4<Avx512>,
) -> f64x4<Avx512> {
_mm256_mask_blend_pd(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>, b: f64x4<Avx512>) -> f64x8<Avx512> {
_mm512_insertf64x4::<1>(_mm512_castpd256_pd512(a.into()), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn split_f64x4(self, a: f64x4<Self>) -> (f64x2<Self>, f64x2<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>) -> (f64x2<Avx512>, f64x2<Avx512>) {
(
_mm256_extractf128_pd::<0>(a.into()).simd_into(token),
_mm256_extractf128_pd::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_f32_f64x4(self, a: f64x4<Self>) -> f32x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x4<Avx512>) -> f32x8<Avx512> {
_mm256_castpd_ps(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_mask64x4(self, val: bool) -> mask64x4<Self> {
mask64x4 {
val: (if val { 15u64 } else { 0 }) as _,
simd: self,
}
}
#[inline(always)]
fn load_array_mask64x4(self, val: [i64; 4usize]) -> mask64x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: [i64; 4usize]) -> mask64x4<Avx512> {
let lanes = crate::transmute::checked_transmute_copy(&val);
mask64x4 {
val: _mm256_movepi64_mask(lanes),
simd: token,
}
}
);
kernel(self, val)
}
#[inline(always)]
fn as_array_mask64x4(self, a: mask64x4<Self>) -> [i64; 4usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: mask64x4<Avx512>) -> [i64; 4usize] {
let lanes = _mm256_movm_epi64(a.val);
crate::transmute::checked_transmute_copy(&lanes)
}
);
kernel(self, a)
}
#[inline(always)]
fn from_bitmask_mask64x4(self, bits: u64) -> mask64x4<Self> {
mask64x4 {
val: (bits & 15u64) as _,
simd: self,
}
}
#[inline(always)]
fn to_bitmask_mask64x4(self, a: mask64x4<Self>) -> u64 {
u64::from((a).val) & 15u64
}
#[inline(always)]
fn set_mask64x4(self, a: &mut mask64x4<Self>, index: usize, value: bool) -> () {
assert!(
index < 4usize,
"mask lane index {index} is out of bounds for {} lanes",
4usize
);
let bit = 1u64 << index;
let bits = u64::from((a).val);
let bits = if value { bits | bit } else { bits & !bit };
*a = mask64x4 {
val: (bits) as _,
simd: self,
};
}
#[inline(always)]
fn and_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
mask64x4 {
val: ((u64::from((a).val) & u64::from((b).val)) & 15u64) as _,
simd: self,
}
}
#[inline(always)]
fn or_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
mask64x4 {
val: ((u64::from((a).val) | u64::from((b).val)) & 15u64) as _,
simd: self,
}
}
#[inline(always)]
fn xor_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
mask64x4 {
val: ((u64::from((a).val) ^ u64::from((b).val)) & 15u64) as _,
simd: self,
}
}
#[inline(always)]
fn not_mask64x4(self, a: mask64x4<Self>) -> mask64x4<Self> {
mask64x4 {
val: ((!u64::from((a).val)) & 15u64) as _,
simd: self,
}
}
#[inline(always)]
fn select_mask64x4(
self,
a: mask64x4<Self>,
b: mask64x4<Self>,
c: mask64x4<Self>,
) -> mask64x4<Self> {
mask64x4 {
val: (((u64::from((a).val) & u64::from((b).val))
| ((!u64::from((a).val)) & u64::from((c).val)))
& 15u64) as _,
simd: self,
}
}
#[inline(always)]
fn simd_eq_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x4<Self> {
mask64x4 {
val: (!u64::from(a.val ^ b.val) & 15u64) as _,
simd: self,
}
}
#[inline(always)]
fn any_true_mask64x4(self, a: mask64x4<Self>) -> bool {
let bits = u64::from((a).val) & 15u64;
bits != 0
}
#[inline(always)]
fn all_true_mask64x4(self, a: mask64x4<Self>) -> bool {
let bits = u64::from((a).val) & 15u64;
bits == 15u64
}
#[inline(always)]
fn any_false_mask64x4(self, a: mask64x4<Self>) -> bool {
let bits = u64::from((a).val) & 15u64;
bits != 15u64
}
#[inline(always)]
fn all_false_mask64x4(self, a: mask64x4<Self>) -> bool {
let bits = u64::from((a).val) & 15u64;
bits == 0
}
#[inline(always)]
fn combine_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x8<Self> {
let bits = (u64::from(a.val) | (u64::from(b.val) << 4usize)) & 255u64;
mask64x8 {
val: bits as _,
simd: self,
}
}
#[inline(always)]
fn split_mask64x4(self, a: mask64x4<Self>) -> (mask64x2<Self>, mask64x2<Self>) {
let bits = u64::from(a.val);
(
mask64x2 {
val: (bits & 3u64) as _,
simd: self,
},
mask64x2 {
val: ((bits >> 2usize) & 3u64) as _,
simd: self,
},
)
}
#[inline(always)]
fn splat_f32x16(self, val: f32) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: f32) -> f32x16<Avx512> {
_mm512_set1_ps(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_f32x16(self, val: [f32; 16usize]) -> f32x16<Self> {
f32x16 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_f32x16(self, val: &[f32; 16usize]) -> f32x16<Self> {
f32x16 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_f32x16(self, a: f32x16<Self>) -> [f32; 16usize] {
crate::transmute::checked_transmute_copy::<__m512, [f32; 16usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_f32x16(self, a: &f32x16<Self>) -> &[f32; 16usize] {
crate::transmute::checked_cast_ref::<__m512, [f32; 16usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_f32x16(self, a: &mut f32x16<Self>) -> &mut [f32; 16usize] {
crate::transmute::checked_cast_mut::<__m512, [f32; 16usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_f32x16(self, a: f32x16<Self>, dest: &mut [f32; 16usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_f32x16(self, a: u8x64<Self>) -> f32x16<Self> {
f32x16 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_f32x16(self, a: f32x16<Self>) -> u8x64<Self> {
u8x64 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_f32x16<const SHIFT: usize>(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f32x16<Avx512>,
b: f32x16<Avx512>,
shift: usize,
) -> f32x16<Avx512> {
if shift >= 16usize {
return b;
}
let idx = _mm512_add_epi8(
_mm512_set_epi8(
63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45,
44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26,
25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
5, 4, 3, 2, 1, 0,
),
_mm512_set1_epi8((shift * 4usize) as i8),
);
let result = _mm512_permutex2var_epi8(
token.cvt_to_bytes_f32x16(a).val.0,
idx,
token.cvt_to_bytes_f32x16(b).val.0,
);
token.cvt_from_bytes_f32x16(u8x64 {
val: crate::support::Aligned512(result),
simd: token,
})
}
);
kernel(self, a, b, SHIFT)
}
#[inline(always)]
fn slide_within_blocks_f32x16<const SHIFT: usize>(
self,
a: f32x16<Self>,
b: f32x16<Self>,
) -> f32x16<Self> {
if SHIFT == 0 {
return a;
}
if SHIFT >= 4usize {
return b;
}
let a = self.cvt_to_bytes_f32x16(a).val.0;
let b = self.cvt_to_bytes_f32x16(b).val.0;
let result = dyn_alignr_512(self, b, a, SHIFT * 4usize);
self.cvt_from_bytes_f32x16(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_within_blocks_f32x16(
self,
a: f32x16<Self>,
indices: u8x64<Self>,
) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>, indices: u8x64<Avx512>) -> f32x16<Avx512> {
let result =
_mm512_shuffle_epi8(token.cvt_to_bytes_f32x16(a).val.0, indices.into());
token.cvt_from_bytes_f32x16(u8x64 {
val: crate::support::Aligned512(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn abs_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>) -> f32x16<Avx512> {
_mm512_andnot_ps(_mm512_set1_ps(-0.0), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>) -> f32x16<Avx512> {
_mm512_xor_ps(a.into(), _mm512_set1_ps(-0.0)).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn sqrt_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>) -> f32x16<Avx512> {
_mm512_sqrt_ps(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn approximate_recip_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>) -> f32x16<Avx512> {
_mm512_rcp14_ps(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn add_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
_mm512_add_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
_mm512_sub_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
_mm512_mul_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn div_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
_mm512_div_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn copysign_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
let mask = _mm512_set1_ps(-0.0);
_mm512_or_ps(
_mm512_and_ps(mask, b.into()),
_mm512_andnot_ps(mask, a.into()),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> mask32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> mask32x16<Avx512> {
mask32x16 {
val: _mm512_cmp_ps_mask::<0i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> mask32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> mask32x16<Avx512> {
mask32x16 {
val: _mm512_cmp_ps_mask::<17i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> mask32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> mask32x16<Avx512> {
mask32x16 {
val: _mm512_cmp_ps_mask::<18i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> mask32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> mask32x16<Avx512> {
mask32x16 {
val: _mm512_cmp_ps_mask::<29i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> mask32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> mask32x16<Avx512> {
mask32x16 {
val: _mm512_cmp_ps_mask::<30i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
_mm512_permutex2var_ps(
a.into(),
_mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
_mm512_permutex2var_ps(
a.into(),
_mm512_setr_epi32(8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
_mm512_permutex2var_ps(
a.into(),
_mm512_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
_mm512_permutex2var_ps(
a.into(),
_mm512_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> (f32x16<Self>, f32x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f32x16<Avx512>,
b: f32x16<Avx512>,
) -> (f32x16<Avx512>, f32x16<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm512_permutex2var_ps(
a,
_mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23),
b,
)
.simd_into(token),
_mm512_permutex2var_ps(
a,
_mm512_setr_epi32(
8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> (f32x16<Self>, f32x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f32x16<Avx512>,
b: f32x16<Avx512>,
) -> (f32x16<Avx512>, f32x16<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm512_permutex2var_ps(
a,
_mm512_setr_epi32(
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30,
),
b,
)
.simd_into(token),
_mm512_permutex2var_ps(
a,
_mm512_setr_epi32(
1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
_mm512_max_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
_mm512_min_ps(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_precise_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
_mm512_range_ps::<5i32>(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_precise_f32x16(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>, b: f32x16<Avx512>) -> f32x16<Avx512> {
_mm512_range_ps::<4i32>(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_add_f32x16(self, a: f32x16<Self>, b: f32x16<Self>, c: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f32x16<Avx512>,
b: f32x16<Avx512>,
c: f32x16<Avx512>,
) -> f32x16<Avx512> {
_mm512_fmadd_ps(a.into(), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn mul_sub_f32x16(self, a: f32x16<Self>, b: f32x16<Self>, c: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f32x16<Avx512>,
b: f32x16<Avx512>,
c: f32x16<Avx512>,
) -> f32x16<Avx512> {
_mm512_fmsub_ps(a.into(), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn floor_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>) -> f32x16<Avx512> {
_mm512_roundscale_ps::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn ceil_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>) -> f32x16<Avx512> {
_mm512_roundscale_ps::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn round_ties_even_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>) -> f32x16<Avx512> {
_mm512_roundscale_ps::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn fract_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
a - self.trunc_f32x16(a)
}
#[inline(always)]
fn trunc_f32x16(self, a: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>) -> f32x16<Avx512> {
_mm512_roundscale_ps::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn select_f32x16(self, a: mask32x16<Self>, b: f32x16<Self>, c: f32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask32x16<Avx512>,
b: f32x16<Avx512>,
c: f32x16<Avx512>,
) -> f32x16<Avx512> {
_mm512_mask_blend_ps(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn split_f32x16(self, a: f32x16<Self>) -> (f32x8<Self>, f32x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>) -> (f32x8<Avx512>, f32x8<Avx512>) {
(
_mm512_castps512_ps256(a.into()).simd_into(token),
_mm512_extractf32x8_ps::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_f64_f32x16(self, a: f32x16<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>) -> f64x8<Avx512> {
_mm512_castps_pd(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_i32_f32x16(self, a: f32x16<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>) -> i32x16<Avx512> {
_mm512_castps_si512(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn load_interleaved_128_f32x16(self, src: &[f32; 16usize]) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, src: &[f32; 16usize]) -> f32x16<Avx512> {
let lanes: __m512 =
crate::transmute::checked_transmute_copy::<[f32; 16usize], __m512>(src);
_mm512_permutexvar_ps(
_mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15),
lanes,
)
.simd_into(token)
}
);
kernel(self, src)
}
#[inline(always)]
fn store_interleaved_128_f32x16(self, a: f32x16<Self>, dest: &mut [f32; 16usize]) -> () {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>, dest: &mut [f32; 16usize]) -> () {
let lanes = _mm512_permutexvar_ps(
_mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15),
a.into(),
);
crate::transmute::checked_transmute_store::<__m512, [f32; 16usize]>(lanes, dest);
}
);
kernel(self, a, dest);
}
#[inline(always)]
fn reinterpret_u8_f32x16(self, a: f32x16<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>) -> u8x64<Avx512> {
_mm512_castps_si512(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u32_f32x16(self, a: f32x16<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>) -> u32x16<Avx512> {
_mm512_castps_si512(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_u32_f32x16(self, a: f32x16<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>) -> u32x16<Avx512> {
_mm512_cvttps_epu32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_u32_precise_f32x16(self, a: f32x16<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>) -> u32x16<Avx512> {
let a = _mm512_max_ps(a.into(), _mm512_setzero_ps());
let mut converted = _mm512_cvttps_epu32(a);
let exceeds_unsigned_range =
_mm512_cmp_ps_mask::<17i32>(_mm512_set1_ps(4294967040.0), a);
converted = _mm512_mask_blend_epi32(
exceeds_unsigned_range,
converted,
_mm512_set1_epi32(u32::MAX.cast_signed()),
);
converted.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_i32_f32x16(self, a: f32x16<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>) -> i32x16<Avx512> {
_mm512_cvttps_epi32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_i32_precise_f32x16(self, a: f32x16<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f32x16<Avx512>) -> i32x16<Avx512> {
let a = a.into();
let in_range = _mm512_cmp_ps_mask::<17i32>(a, _mm512_set1_ps(2147483648.0));
let mut converted =
_mm512_mask_cvttps_epi32(_mm512_set1_epi32(i32::MAX), in_range, a);
let is_not_nan = _mm512_cmp_ps_mask::<7i32>(a, a);
converted = _mm512_mask_blend_epi32(is_not_nan, _mm512_setzero_si512(), converted);
converted.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_i8x64(self, val: i8) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: i8) -> i8x64<Avx512> {
_mm512_set1_epi8(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_i8x64(self, val: [i8; 64usize]) -> i8x64<Self> {
i8x64 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_i8x64(self, val: &[i8; 64usize]) -> i8x64<Self> {
i8x64 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_i8x64(self, a: i8x64<Self>) -> [i8; 64usize] {
crate::transmute::checked_transmute_copy::<__m512i, [i8; 64usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_i8x64(self, a: &i8x64<Self>) -> &[i8; 64usize] {
crate::transmute::checked_cast_ref::<__m512i, [i8; 64usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_i8x64(self, a: &mut i8x64<Self>) -> &mut [i8; 64usize] {
crate::transmute::checked_cast_mut::<__m512i, [i8; 64usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_i8x64(self, a: i8x64<Self>, dest: &mut [i8; 64usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_i8x64(self, a: u8x64<Self>) -> i8x64<Self> {
i8x64 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_i8x64(self, a: i8x64<Self>) -> u8x64<Self> {
u8x64 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_i8x64<const SHIFT: usize>(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i8x64<Avx512>,
b: i8x64<Avx512>,
shift: usize,
) -> i8x64<Avx512> {
if shift >= 64usize {
return b;
}
let idx = _mm512_add_epi8(
_mm512_set_epi8(
63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45,
44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26,
25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
5, 4, 3, 2, 1, 0,
),
_mm512_set1_epi8((shift) as i8),
);
let result = _mm512_permutex2var_epi8(
token.cvt_to_bytes_i8x64(a).val.0,
idx,
token.cvt_to_bytes_i8x64(b).val.0,
);
token.cvt_from_bytes_i8x64(u8x64 {
val: crate::support::Aligned512(result),
simd: token,
})
}
);
kernel(self, a, b, SHIFT)
}
#[inline(always)]
fn slide_within_blocks_i8x64<const SHIFT: usize>(
self,
a: i8x64<Self>,
b: i8x64<Self>,
) -> i8x64<Self> {
if SHIFT == 0 {
return a;
}
if SHIFT >= 16usize {
return b;
}
let a = self.cvt_to_bytes_i8x64(a).val.0;
let b = self.cvt_to_bytes_i8x64(b).val.0;
let result = dyn_alignr_512(self, b, a, SHIFT);
self.cvt_from_bytes_i8x64(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_within_blocks_i8x64(self, a: i8x64<Self>, indices: u8x64<Self>) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, indices: u8x64<Avx512>) -> i8x64<Avx512> {
let result = _mm512_shuffle_epi8(token.cvt_to_bytes_i8x64(a).val.0, indices.into());
token.cvt_from_bytes_i8x64(u8x64 {
val: crate::support::Aligned512(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
_mm512_add_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
_mm512_sub_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
let dst_even = _mm512_mullo_epi16(a.into(), b.into());
let dst_odd = _mm512_mullo_epi16(
_mm512_srli_epi16::<8>(a.into()),
_mm512_srli_epi16::<8>(b.into()),
);
_mm512_or_si512(
_mm512_slli_epi16(dst_odd, 8),
_mm512_and_si512(dst_even, _mm512_set1_epi16(0xFF)),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
_mm512_and_si512(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
_mm512_or_si512(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
_mm512_xor_si512(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i8x64(self, a: i8x64<Self>) -> i8x64<Self> {
a ^ !0
}
#[inline(always)]
fn shl_i8x64(self, a: i8x64<Self>, shift: u32) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, shift: u32) -> i8x64<Avx512> {
let val = a.into();
let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
let lo_16 = _mm512_unpacklo_epi8(
val,
_mm512_movm_epi8(_mm512_cmpgt_epi8_mask(_mm512_setzero_si512(), val)),
);
let hi_16 = _mm512_unpackhi_epi8(
val,
_mm512_movm_epi8(_mm512_cmpgt_epi8_mask(_mm512_setzero_si512(), val)),
);
let lo_shifted = _mm512_sll_epi16(lo_16, shift_count);
let hi_shifted = _mm512_sll_epi16(hi_16, shift_count);
_mm512_packs_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
let val = a.into();
let counts = b.into();
let zero = _mm512_setzero_si512();
let value_extend = zero;
let lo_values = _mm512_unpacklo_epi8(val, value_extend);
let hi_values = _mm512_unpackhi_epi8(val, value_extend);
let lo_counts = _mm512_unpacklo_epi8(counts, zero);
let hi_counts = _mm512_unpackhi_epi8(counts, zero);
let byte_mask = _mm512_set1_epi16(0x00ff);
let lo_shifted =
_mm512_and_si512(_mm512_sllv_epi16(lo_values, lo_counts), byte_mask);
let hi_shifted =
_mm512_and_si512(_mm512_sllv_epi16(hi_values, hi_counts), byte_mask);
_mm512_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_i8x64(self, a: i8x64<Self>, shift: u32) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, shift: u32) -> i8x64<Avx512> {
let val = a.into();
let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
let lo_16 = _mm512_unpacklo_epi8(
val,
_mm512_movm_epi8(_mm512_cmpgt_epi8_mask(_mm512_setzero_si512(), val)),
);
let hi_16 = _mm512_unpackhi_epi8(
val,
_mm512_movm_epi8(_mm512_cmpgt_epi8_mask(_mm512_setzero_si512(), val)),
);
let lo_shifted = _mm512_sra_epi16(lo_16, shift_count);
let hi_shifted = _mm512_sra_epi16(hi_16, shift_count);
_mm512_packs_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
let val = a.into();
let counts = b.into();
let zero = _mm512_setzero_si512();
let value_extend = _mm512_movm_epi8(_mm512_cmpgt_epi8_mask(zero, val));
let lo_values = _mm512_unpacklo_epi8(val, value_extend);
let hi_values = _mm512_unpackhi_epi8(val, value_extend);
let lo_counts = _mm512_unpacklo_epi8(counts, zero);
let hi_counts = _mm512_unpackhi_epi8(counts, zero);
let byte_mask = _mm512_set1_epi16(0x00ff);
let lo_shifted =
_mm512_and_si512(_mm512_srav_epi16(lo_values, lo_counts), byte_mask);
let hi_shifted =
_mm512_and_si512(_mm512_srav_epi16(hi_values, hi_counts), byte_mask);
_mm512_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> mask8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> mask8x64<Avx512> {
mask8x64 {
val: _mm512_cmpeq_epi8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> mask8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> mask8x64<Avx512> {
mask8x64 {
val: _mm512_cmplt_epi8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> mask8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> mask8x64<Avx512> {
mask8x64 {
val: _mm512_cmple_epi8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> mask8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> mask8x64<Avx512> {
mask8x64 {
val: _mm512_cmpge_epi8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> mask8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> mask8x64<Avx512> {
mask8x64 {
val: _mm512_cmpgt_epi8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
_mm512_permutex2var_epi8(
a.into(),
_mm512_set_epi8(
95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, 86,
22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, 77, 13,
76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3,
66, 2, 65, 1, 64, 0,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
_mm512_permutex2var_epi8(
a.into(),
_mm512_set_epi8(
127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56,
119, 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48,
111, 47, 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40,
103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
_mm512_permutex2var_epi8(
a.into(),
_mm512_set_epi8(
126, 124, 122, 120, 118, 116, 114, 112, 110, 108, 106, 104, 102, 100, 98,
96, 94, 92, 90, 88, 86, 84, 82, 80, 78, 76, 74, 72, 70, 68, 66, 64, 62, 60,
58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28, 26, 24, 22,
20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
_mm512_permutex2var_epi8(
a.into(),
_mm512_set_epi8(
127, 125, 123, 121, 119, 117, 115, 113, 111, 109, 107, 105, 103, 101, 99,
97, 95, 93, 91, 89, 87, 85, 83, 81, 79, 77, 75, 73, 71, 69, 67, 65, 63, 61,
59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29, 27, 25, 23,
21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> (i8x64<Self>, i8x64<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i8x64<Avx512>,
b: i8x64<Avx512>,
) -> (i8x64<Avx512>, i8x64<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm512_permutex2var_epi8(
a,
_mm512_set_epi8(
95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23,
86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14,
77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68,
4, 67, 3, 66, 2, 65, 1, 64, 0,
),
b,
)
.simd_into(token),
_mm512_permutex2var_epi8(
a,
_mm512_set_epi8(
127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56,
119, 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48,
111, 47, 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40,
103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> (i8x64<Self>, i8x64<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i8x64<Avx512>,
b: i8x64<Avx512>,
) -> (i8x64<Avx512>, i8x64<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm512_permutex2var_epi8(
a,
_mm512_set_epi8(
126, 124, 122, 120, 118, 116, 114, 112, 110, 108, 106, 104, 102, 100,
98, 96, 94, 92, 90, 88, 86, 84, 82, 80, 78, 76, 74, 72, 70, 68, 66, 64,
62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28,
26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0,
),
b,
)
.simd_into(token),
_mm512_permutex2var_epi8(
a,
_mm512_set_epi8(
127, 125, 123, 121, 119, 117, 115, 113, 111, 109, 107, 105, 103, 101,
99, 97, 95, 93, 91, 89, 87, 85, 83, 81, 79, 77, 75, 73, 71, 69, 67, 65,
63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29,
27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_i8x64(self, a: mask8x64<Self>, b: i8x64<Self>, c: i8x64<Self>) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask8x64<Avx512>,
b: i8x64<Avx512>,
c: i8x64<Avx512>,
) -> i8x64<Avx512> {
_mm512_mask_blend_epi8(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn min_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
_mm512_min_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_i8x64(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>, b: i8x64<Avx512>) -> i8x64<Avx512> {
_mm512_max_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn split_i8x64(self, a: i8x64<Self>) -> (i8x32<Self>, i8x32<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>) -> (i8x32<Avx512>, i8x32<Avx512>) {
(
_mm512_castsi512_si256(a.into()).simd_into(token),
_mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_i8x64(self, a: i8x64<Self>) -> i8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>) -> i8x64<Avx512> {
_mm512_sub_epi8(_mm512_setzero_si512(), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u8_i8x64(self, a: i8x64<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>) -> u8x64<Avx512> {
__m512i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u32_i8x64(self, a: i8x64<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i8x64<Avx512>) -> u32x16<Avx512> {
__m512i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_u8x64(self, val: u8) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: u8) -> u8x64<Avx512> {
_mm512_set1_epi8(val.cast_signed()).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_u8x64(self, val: [u8; 64usize]) -> u8x64<Self> {
u8x64 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_u8x64(self, val: &[u8; 64usize]) -> u8x64<Self> {
u8x64 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_u8x64(self, a: u8x64<Self>) -> [u8; 64usize] {
crate::transmute::checked_transmute_copy::<__m512i, [u8; 64usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_u8x64(self, a: &u8x64<Self>) -> &[u8; 64usize] {
crate::transmute::checked_cast_ref::<__m512i, [u8; 64usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_u8x64(self, a: &mut u8x64<Self>) -> &mut [u8; 64usize] {
crate::transmute::checked_cast_mut::<__m512i, [u8; 64usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_u8x64(self, a: u8x64<Self>, dest: &mut [u8; 64usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_u8x64(self, a: u8x64<Self>) -> u8x64<Self> {
u8x64 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_u8x64(self, a: u8x64<Self>) -> u8x64<Self> {
u8x64 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_u8x64<const SHIFT: usize>(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u8x64<Avx512>,
b: u8x64<Avx512>,
shift: usize,
) -> u8x64<Avx512> {
if shift >= 64usize {
return b;
}
let idx = _mm512_add_epi8(
_mm512_set_epi8(
63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45,
44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26,
25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
5, 4, 3, 2, 1, 0,
),
_mm512_set1_epi8((shift) as i8),
);
let result = _mm512_permutex2var_epi8(
token.cvt_to_bytes_u8x64(a).val.0,
idx,
token.cvt_to_bytes_u8x64(b).val.0,
);
token.cvt_from_bytes_u8x64(u8x64 {
val: crate::support::Aligned512(result),
simd: token,
})
}
);
kernel(self, a, b, SHIFT)
}
#[inline(always)]
fn slide_within_blocks_u8x64<const SHIFT: usize>(
self,
a: u8x64<Self>,
b: u8x64<Self>,
) -> u8x64<Self> {
if SHIFT == 0 {
return a;
}
if SHIFT >= 16usize {
return b;
}
let a = self.cvt_to_bytes_u8x64(a).val.0;
let b = self.cvt_to_bytes_u8x64(b).val.0;
let result = dyn_alignr_512(self, b, a, SHIFT);
self.cvt_from_bytes_u8x64(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_within_blocks_u8x64(self, a: u8x64<Self>, indices: u8x64<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, indices: u8x64<Avx512>) -> u8x64<Avx512> {
let result = _mm512_shuffle_epi8(token.cvt_to_bytes_u8x64(a).val.0, indices.into());
token.cvt_from_bytes_u8x64(u8x64 {
val: crate::support::Aligned512(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
_mm512_add_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
_mm512_sub_epi8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
let dst_even = _mm512_mullo_epi16(a.into(), b.into());
let dst_odd = _mm512_mullo_epi16(
_mm512_srli_epi16::<8>(a.into()),
_mm512_srli_epi16::<8>(b.into()),
);
_mm512_or_si512(
_mm512_slli_epi16(dst_odd, 8),
_mm512_and_si512(dst_even, _mm512_set1_epi16(0xFF)),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
_mm512_and_si512(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
_mm512_or_si512(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
_mm512_xor_si512(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u8x64(self, a: u8x64<Self>) -> u8x64<Self> {
a ^ !0
}
#[inline(always)]
fn shl_u8x64(self, a: u8x64<Self>, shift: u32) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, shift: u32) -> u8x64<Avx512> {
let val = a.into();
let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
let lo_16 = _mm512_unpacklo_epi8(val, _mm512_setzero_si512());
let hi_16 = _mm512_unpackhi_epi8(val, _mm512_setzero_si512());
let lo_shifted = _mm512_sll_epi16(lo_16, shift_count);
let hi_shifted = _mm512_sll_epi16(hi_16, shift_count);
_mm512_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
let val = a.into();
let counts = b.into();
let zero = _mm512_setzero_si512();
let value_extend = zero;
let lo_values = _mm512_unpacklo_epi8(val, value_extend);
let hi_values = _mm512_unpackhi_epi8(val, value_extend);
let lo_counts = _mm512_unpacklo_epi8(counts, zero);
let hi_counts = _mm512_unpackhi_epi8(counts, zero);
let byte_mask = _mm512_set1_epi16(0x00ff);
let lo_shifted =
_mm512_and_si512(_mm512_sllv_epi16(lo_values, lo_counts), byte_mask);
let hi_shifted =
_mm512_and_si512(_mm512_sllv_epi16(hi_values, hi_counts), byte_mask);
_mm512_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_u8x64(self, a: u8x64<Self>, shift: u32) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, shift: u32) -> u8x64<Avx512> {
let val = a.into();
let shift_count = _mm_cvtsi32_si128(shift.cast_signed());
let lo_16 = _mm512_unpacklo_epi8(val, _mm512_setzero_si512());
let hi_16 = _mm512_unpackhi_epi8(val, _mm512_setzero_si512());
let lo_shifted = _mm512_srl_epi16(lo_16, shift_count);
let hi_shifted = _mm512_srl_epi16(hi_16, shift_count);
_mm512_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
let val = a.into();
let counts = b.into();
let zero = _mm512_setzero_si512();
let value_extend = zero;
let lo_values = _mm512_unpacklo_epi8(val, value_extend);
let hi_values = _mm512_unpackhi_epi8(val, value_extend);
let lo_counts = _mm512_unpacklo_epi8(counts, zero);
let hi_counts = _mm512_unpackhi_epi8(counts, zero);
let byte_mask = _mm512_set1_epi16(0x00ff);
let lo_shifted =
_mm512_and_si512(_mm512_srlv_epi16(lo_values, lo_counts), byte_mask);
let hi_shifted =
_mm512_and_si512(_mm512_srlv_epi16(hi_values, hi_counts), byte_mask);
_mm512_packus_epi16(lo_shifted, hi_shifted).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> mask8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> mask8x64<Avx512> {
mask8x64 {
val: _mm512_cmpeq_epu8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> mask8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> mask8x64<Avx512> {
mask8x64 {
val: _mm512_cmplt_epu8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> mask8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> mask8x64<Avx512> {
mask8x64 {
val: _mm512_cmple_epu8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> mask8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> mask8x64<Avx512> {
mask8x64 {
val: _mm512_cmpge_epu8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> mask8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> mask8x64<Avx512> {
mask8x64 {
val: _mm512_cmpgt_epu8_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
_mm512_permutex2var_epi8(
a.into(),
_mm512_set_epi8(
95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, 86,
22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, 77, 13,
76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3,
66, 2, 65, 1, 64, 0,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
_mm512_permutex2var_epi8(
a.into(),
_mm512_set_epi8(
127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56,
119, 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48,
111, 47, 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40,
103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
_mm512_permutex2var_epi8(
a.into(),
_mm512_set_epi8(
126, 124, 122, 120, 118, 116, 114, 112, 110, 108, 106, 104, 102, 100, 98,
96, 94, 92, 90, 88, 86, 84, 82, 80, 78, 76, 74, 72, 70, 68, 66, 64, 62, 60,
58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28, 26, 24, 22,
20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
_mm512_permutex2var_epi8(
a.into(),
_mm512_set_epi8(
127, 125, 123, 121, 119, 117, 115, 113, 111, 109, 107, 105, 103, 101, 99,
97, 95, 93, 91, 89, 87, 85, 83, 81, 79, 77, 75, 73, 71, 69, 67, 65, 63, 61,
59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29, 27, 25, 23,
21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> (u8x64<Self>, u8x64<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u8x64<Avx512>,
b: u8x64<Avx512>,
) -> (u8x64<Avx512>, u8x64<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm512_permutex2var_epi8(
a,
_mm512_set_epi8(
95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23,
86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14,
77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68,
4, 67, 3, 66, 2, 65, 1, 64, 0,
),
b,
)
.simd_into(token),
_mm512_permutex2var_epi8(
a,
_mm512_set_epi8(
127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56,
119, 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48,
111, 47, 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40,
103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> (u8x64<Self>, u8x64<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u8x64<Avx512>,
b: u8x64<Avx512>,
) -> (u8x64<Avx512>, u8x64<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm512_permutex2var_epi8(
a,
_mm512_set_epi8(
126, 124, 122, 120, 118, 116, 114, 112, 110, 108, 106, 104, 102, 100,
98, 96, 94, 92, 90, 88, 86, 84, 82, 80, 78, 76, 74, 72, 70, 68, 66, 64,
62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28,
26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0,
),
b,
)
.simd_into(token),
_mm512_permutex2var_epi8(
a,
_mm512_set_epi8(
127, 125, 123, 121, 119, 117, 115, 113, 111, 109, 107, 105, 103, 101,
99, 97, 95, 93, 91, 89, 87, 85, 83, 81, 79, 77, 75, 73, 71, 69, 67, 65,
63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29,
27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_u8x64(self, a: mask8x64<Self>, b: u8x64<Self>, c: u8x64<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask8x64<Avx512>,
b: u8x64<Avx512>,
c: u8x64<Avx512>,
) -> u8x64<Avx512> {
_mm512_mask_blend_epi8(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn min_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
_mm512_min_epu8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_u8x64(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, b: u8x64<Avx512>) -> u8x64<Avx512> {
_mm512_max_epu8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn split_u8x64(self, a: u8x64<Self>) -> (u8x32<Self>, u8x32<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>) -> (u8x32<Avx512>, u8x32<Avx512>) {
(
_mm512_castsi512_si256(a.into()).simd_into(token),
_mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn load_interleaved_128_u8x64(self, src: &[u8; 64usize]) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, src: &[u8; 64usize]) -> u8x64<Avx512> {
let lanes: __m512i =
crate::transmute::checked_transmute_copy::<[u8; 64usize], __m512i>(src);
_mm512_permutexvar_epi8(
_mm512_set_epi8(
63, 59, 55, 51, 47, 43, 39, 35, 31, 27, 23, 19, 15, 11, 7, 3, 62, 58, 54,
50, 46, 42, 38, 34, 30, 26, 22, 18, 14, 10, 6, 2, 61, 57, 53, 49, 45, 41,
37, 33, 29, 25, 21, 17, 13, 9, 5, 1, 60, 56, 52, 48, 44, 40, 36, 32, 28,
24, 20, 16, 12, 8, 4, 0,
),
lanes,
)
.simd_into(token)
}
);
kernel(self, src)
}
#[inline(always)]
fn store_interleaved_128_u8x64(self, a: u8x64<Self>, dest: &mut [u8; 64usize]) -> () {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>, dest: &mut [u8; 64usize]) -> () {
let lanes = _mm512_permutexvar_epi8(
_mm512_set_epi8(
63, 47, 31, 15, 62, 46, 30, 14, 61, 45, 29, 13, 60, 44, 28, 12, 59, 43, 27,
11, 58, 42, 26, 10, 57, 41, 25, 9, 56, 40, 24, 8, 55, 39, 23, 7, 54, 38,
22, 6, 53, 37, 21, 5, 52, 36, 20, 4, 51, 35, 19, 3, 50, 34, 18, 2, 49, 33,
17, 1, 48, 32, 16, 0,
),
a.into(),
);
crate::transmute::checked_transmute_store::<__m512i, [u8; 64usize]>(lanes, dest);
}
);
kernel(self, a, dest);
}
#[inline(always)]
fn reinterpret_u32_u8x64(self, a: u8x64<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u8x64<Avx512>) -> u32x16<Avx512> {
__m512i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_mask8x64(self, val: bool) -> mask8x64<Self> {
mask8x64 {
val: if val { u64::MAX } else { 0 },
simd: self,
}
}
#[inline(always)]
fn load_array_mask8x64(self, val: [i8; 64usize]) -> mask8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: [i8; 64usize]) -> mask8x64<Avx512> {
let lanes = crate::transmute::checked_transmute_copy(&val);
mask8x64 {
val: _mm512_movepi8_mask(lanes),
simd: token,
}
}
);
kernel(self, val)
}
#[inline(always)]
fn as_array_mask8x64(self, a: mask8x64<Self>) -> [i8; 64usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: mask8x64<Avx512>) -> [i8; 64usize] {
let lanes = _mm512_movm_epi8(a.val);
crate::transmute::checked_transmute_copy(&lanes)
}
);
kernel(self, a)
}
#[inline(always)]
fn from_bitmask_mask8x64(self, bits: u64) -> mask8x64<Self> {
mask8x64 {
val: bits & u64::MAX,
simd: self,
}
}
#[inline(always)]
fn to_bitmask_mask8x64(self, a: mask8x64<Self>) -> u64 {
u64::from((a).val) & u64::MAX
}
#[inline(always)]
fn set_mask8x64(self, a: &mut mask8x64<Self>, index: usize, value: bool) -> () {
assert!(
index < 64usize,
"mask lane index {index} is out of bounds for {} lanes",
64usize
);
let bit = 1u64 << index;
let bits = u64::from((a).val);
let bits = if value { bits | bit } else { bits & !bit };
*a = mask8x64 {
val: bits,
simd: self,
};
}
#[inline(always)]
fn and_mask8x64(self, a: mask8x64<Self>, b: mask8x64<Self>) -> mask8x64<Self> {
mask8x64 {
val: (u64::from((a).val) & u64::from((b).val)) & u64::MAX,
simd: self,
}
}
#[inline(always)]
fn or_mask8x64(self, a: mask8x64<Self>, b: mask8x64<Self>) -> mask8x64<Self> {
mask8x64 {
val: (u64::from((a).val) | u64::from((b).val)) & u64::MAX,
simd: self,
}
}
#[inline(always)]
fn xor_mask8x64(self, a: mask8x64<Self>, b: mask8x64<Self>) -> mask8x64<Self> {
mask8x64 {
val: (u64::from((a).val) ^ u64::from((b).val)) & u64::MAX,
simd: self,
}
}
#[inline(always)]
fn not_mask8x64(self, a: mask8x64<Self>) -> mask8x64<Self> {
mask8x64 {
val: (!u64::from((a).val)) & u64::MAX,
simd: self,
}
}
#[inline(always)]
fn select_mask8x64(
self,
a: mask8x64<Self>,
b: mask8x64<Self>,
c: mask8x64<Self>,
) -> mask8x64<Self> {
mask8x64 {
val: ((u64::from((a).val) & u64::from((b).val))
| ((!u64::from((a).val)) & u64::from((c).val)))
& u64::MAX,
simd: self,
}
}
#[inline(always)]
fn simd_eq_mask8x64(self, a: mask8x64<Self>, b: mask8x64<Self>) -> mask8x64<Self> {
mask8x64 {
val: !u64::from(a.val ^ b.val) & u64::MAX,
simd: self,
}
}
#[inline(always)]
fn any_true_mask8x64(self, a: mask8x64<Self>) -> bool {
let bits = u64::from((a).val) & u64::MAX;
bits != 0
}
#[inline(always)]
fn all_true_mask8x64(self, a: mask8x64<Self>) -> bool {
let bits = u64::from((a).val) & u64::MAX;
bits == u64::MAX
}
#[inline(always)]
fn any_false_mask8x64(self, a: mask8x64<Self>) -> bool {
let bits = u64::from((a).val) & u64::MAX;
bits != u64::MAX
}
#[inline(always)]
fn all_false_mask8x64(self, a: mask8x64<Self>) -> bool {
let bits = u64::from((a).val) & u64::MAX;
bits == 0
}
#[inline(always)]
fn split_mask8x64(self, a: mask8x64<Self>) -> (mask8x32<Self>, mask8x32<Self>) {
let bits = u64::from(a.val);
(
mask8x32 {
val: (bits & 4294967295u64) as _,
simd: self,
},
mask8x32 {
val: ((bits >> 32usize) & 4294967295u64) as _,
simd: self,
},
)
}
#[inline(always)]
fn splat_i16x32(self, val: i16) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: i16) -> i16x32<Avx512> {
_mm512_set1_epi16(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_i16x32(self, val: [i16; 32usize]) -> i16x32<Self> {
i16x32 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_i16x32(self, val: &[i16; 32usize]) -> i16x32<Self> {
i16x32 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_i16x32(self, a: i16x32<Self>) -> [i16; 32usize] {
crate::transmute::checked_transmute_copy::<__m512i, [i16; 32usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_i16x32(self, a: &i16x32<Self>) -> &[i16; 32usize] {
crate::transmute::checked_cast_ref::<__m512i, [i16; 32usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_i16x32(self, a: &mut i16x32<Self>) -> &mut [i16; 32usize] {
crate::transmute::checked_cast_mut::<__m512i, [i16; 32usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_i16x32(self, a: i16x32<Self>, dest: &mut [i16; 32usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_i16x32(self, a: u8x64<Self>) -> i16x32<Self> {
i16x32 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_i16x32(self, a: i16x32<Self>) -> u8x64<Self> {
u8x64 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_i16x32<const SHIFT: usize>(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i16x32<Avx512>,
b: i16x32<Avx512>,
shift: usize,
) -> i16x32<Avx512> {
if shift >= 32usize {
return b;
}
let idx = _mm512_add_epi8(
_mm512_set_epi8(
63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45,
44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26,
25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
5, 4, 3, 2, 1, 0,
),
_mm512_set1_epi8((shift * 2usize) as i8),
);
let result = _mm512_permutex2var_epi8(
token.cvt_to_bytes_i16x32(a).val.0,
idx,
token.cvt_to_bytes_i16x32(b).val.0,
);
token.cvt_from_bytes_i16x32(u8x64 {
val: crate::support::Aligned512(result),
simd: token,
})
}
);
kernel(self, a, b, SHIFT)
}
#[inline(always)]
fn slide_within_blocks_i16x32<const SHIFT: usize>(
self,
a: i16x32<Self>,
b: i16x32<Self>,
) -> i16x32<Self> {
if SHIFT == 0 {
return a;
}
if SHIFT >= 8usize {
return b;
}
let a = self.cvt_to_bytes_i16x32(a).val.0;
let b = self.cvt_to_bytes_i16x32(b).val.0;
let result = dyn_alignr_512(self, b, a, SHIFT * 2usize);
self.cvt_from_bytes_i16x32(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_within_blocks_i16x32(
self,
a: i16x32<Self>,
indices: u8x64<Self>,
) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, indices: u8x64<Avx512>) -> i16x32<Avx512> {
let result =
_mm512_shuffle_epi8(token.cvt_to_bytes_i16x32(a).val.0, indices.into());
token.cvt_from_bytes_i16x32(u8x64 {
val: crate::support::Aligned512(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
_mm512_add_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
_mm512_sub_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
_mm512_mullo_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
_mm512_and_si512(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
_mm512_or_si512(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
_mm512_xor_si512(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i16x32(self, a: i16x32<Self>) -> i16x32<Self> {
a ^ !0
}
#[inline(always)]
fn shl_i16x32(self, a: i16x32<Self>, shift: u32) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, shift: u32) -> i16x32<Avx512> {
_mm512_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
_mm512_sllv_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_i16x32(self, a: i16x32<Self>, shift: u32) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, shift: u32) -> i16x32<Avx512> {
_mm512_sra_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
_mm512_srav_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> mask16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> mask16x32<Avx512> {
mask16x32 {
val: _mm512_cmpeq_epi16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> mask16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> mask16x32<Avx512> {
mask16x32 {
val: _mm512_cmplt_epi16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> mask16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> mask16x32<Avx512> {
mask16x32 {
val: _mm512_cmple_epi16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> mask16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> mask16x32<Avx512> {
mask16x32 {
val: _mm512_cmpge_epi16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> mask16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> mask16x32<Avx512> {
mask16x32 {
val: _mm512_cmpgt_epi16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
_mm512_permutex2var_epi16(
a.into(),
_mm512_set_epi16(
47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6,
37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
_mm512_permutex2var_epi16(
a.into(),
_mm512_set_epi16(
63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54,
22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
_mm512_permutex2var_epi16(
a.into(),
_mm512_set_epi16(
62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28, 26,
24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
_mm512_permutex2var_epi16(
a.into(),
_mm512_set_epi16(
63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29, 27,
25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> (i16x32<Self>, i16x32<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i16x32<Avx512>,
b: i16x32<Avx512>,
) -> (i16x32<Avx512>, i16x32<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm512_permutex2var_epi16(
a,
_mm512_set_epi16(
47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7,
38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0,
),
b,
)
.simd_into(token),
_mm512_permutex2var_epi16(
a,
_mm512_set_epi16(
63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23,
54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> (i16x32<Self>, i16x32<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i16x32<Avx512>,
b: i16x32<Avx512>,
) -> (i16x32<Avx512>, i16x32<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm512_permutex2var_epi16(
a,
_mm512_set_epi16(
62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28,
26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0,
),
b,
)
.simd_into(token),
_mm512_permutex2var_epi16(
a,
_mm512_set_epi16(
63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29,
27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_i16x32(self, a: mask16x32<Self>, b: i16x32<Self>, c: i16x32<Self>) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask16x32<Avx512>,
b: i16x32<Avx512>,
c: i16x32<Avx512>,
) -> i16x32<Avx512> {
_mm512_mask_blend_epi16(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn min_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
_mm512_min_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_i16x32(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>, b: i16x32<Avx512>) -> i16x32<Avx512> {
_mm512_max_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn split_i16x32(self, a: i16x32<Self>) -> (i16x16<Self>, i16x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>) -> (i16x16<Avx512>, i16x16<Avx512>) {
(
_mm512_castsi512_si256(a.into()).simd_into(token),
_mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_i16x32(self, a: i16x32<Self>) -> i16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>) -> i16x32<Avx512> {
_mm512_sub_epi16(_mm512_setzero_si512(), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u8_i16x32(self, a: i16x32<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>) -> u8x64<Avx512> {
__m512i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u32_i16x32(self, a: i16x32<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i16x32<Avx512>) -> u32x16<Avx512> {
__m512i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_u16x32(self, val: u16) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: u16) -> u16x32<Avx512> {
_mm512_set1_epi16(val.cast_signed()).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_u16x32(self, val: [u16; 32usize]) -> u16x32<Self> {
u16x32 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_u16x32(self, val: &[u16; 32usize]) -> u16x32<Self> {
u16x32 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_u16x32(self, a: u16x32<Self>) -> [u16; 32usize] {
crate::transmute::checked_transmute_copy::<__m512i, [u16; 32usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_u16x32(self, a: &u16x32<Self>) -> &[u16; 32usize] {
crate::transmute::checked_cast_ref::<__m512i, [u16; 32usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_u16x32(self, a: &mut u16x32<Self>) -> &mut [u16; 32usize] {
crate::transmute::checked_cast_mut::<__m512i, [u16; 32usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_u16x32(self, a: u16x32<Self>, dest: &mut [u16; 32usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_u16x32(self, a: u8x64<Self>) -> u16x32<Self> {
u16x32 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_u16x32(self, a: u16x32<Self>) -> u8x64<Self> {
u8x64 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_u16x32<const SHIFT: usize>(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u16x32<Avx512>,
b: u16x32<Avx512>,
shift: usize,
) -> u16x32<Avx512> {
if shift >= 32usize {
return b;
}
let idx = _mm512_add_epi8(
_mm512_set_epi8(
63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45,
44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26,
25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
5, 4, 3, 2, 1, 0,
),
_mm512_set1_epi8((shift * 2usize) as i8),
);
let result = _mm512_permutex2var_epi8(
token.cvt_to_bytes_u16x32(a).val.0,
idx,
token.cvt_to_bytes_u16x32(b).val.0,
);
token.cvt_from_bytes_u16x32(u8x64 {
val: crate::support::Aligned512(result),
simd: token,
})
}
);
kernel(self, a, b, SHIFT)
}
#[inline(always)]
fn slide_within_blocks_u16x32<const SHIFT: usize>(
self,
a: u16x32<Self>,
b: u16x32<Self>,
) -> u16x32<Self> {
if SHIFT == 0 {
return a;
}
if SHIFT >= 8usize {
return b;
}
let a = self.cvt_to_bytes_u16x32(a).val.0;
let b = self.cvt_to_bytes_u16x32(b).val.0;
let result = dyn_alignr_512(self, b, a, SHIFT * 2usize);
self.cvt_from_bytes_u16x32(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_within_blocks_u16x32(
self,
a: u16x32<Self>,
indices: u8x64<Self>,
) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, indices: u8x64<Avx512>) -> u16x32<Avx512> {
let result =
_mm512_shuffle_epi8(token.cvt_to_bytes_u16x32(a).val.0, indices.into());
token.cvt_from_bytes_u16x32(u8x64 {
val: crate::support::Aligned512(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
_mm512_add_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
_mm512_sub_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
_mm512_mullo_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
_mm512_and_si512(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
_mm512_or_si512(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
_mm512_xor_si512(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u16x32(self, a: u16x32<Self>) -> u16x32<Self> {
a ^ !0
}
#[inline(always)]
fn shl_u16x32(self, a: u16x32<Self>, shift: u32) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, shift: u32) -> u16x32<Avx512> {
_mm512_sll_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
_mm512_sllv_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_u16x32(self, a: u16x32<Self>, shift: u32) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, shift: u32) -> u16x32<Avx512> {
_mm512_srl_epi16(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
_mm512_srlv_epi16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> mask16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> mask16x32<Avx512> {
mask16x32 {
val: _mm512_cmpeq_epu16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> mask16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> mask16x32<Avx512> {
mask16x32 {
val: _mm512_cmplt_epu16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> mask16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> mask16x32<Avx512> {
mask16x32 {
val: _mm512_cmple_epu16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> mask16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> mask16x32<Avx512> {
mask16x32 {
val: _mm512_cmpge_epu16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> mask16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> mask16x32<Avx512> {
mask16x32 {
val: _mm512_cmpgt_epu16_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
_mm512_permutex2var_epi16(
a.into(),
_mm512_set_epi16(
47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6,
37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
_mm512_permutex2var_epi16(
a.into(),
_mm512_set_epi16(
63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54,
22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
_mm512_permutex2var_epi16(
a.into(),
_mm512_set_epi16(
62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28, 26,
24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
_mm512_permutex2var_epi16(
a.into(),
_mm512_set_epi16(
63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29, 27,
25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1,
),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> (u16x32<Self>, u16x32<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u16x32<Avx512>,
b: u16x32<Avx512>,
) -> (u16x32<Avx512>, u16x32<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm512_permutex2var_epi16(
a,
_mm512_set_epi16(
47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7,
38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0,
),
b,
)
.simd_into(token),
_mm512_permutex2var_epi16(
a,
_mm512_set_epi16(
63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23,
54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> (u16x32<Self>, u16x32<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u16x32<Avx512>,
b: u16x32<Avx512>,
) -> (u16x32<Avx512>, u16x32<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm512_permutex2var_epi16(
a,
_mm512_set_epi16(
62, 60, 58, 56, 54, 52, 50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28,
26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0,
),
b,
)
.simd_into(token),
_mm512_permutex2var_epi16(
a,
_mm512_set_epi16(
63, 61, 59, 57, 55, 53, 51, 49, 47, 45, 43, 41, 39, 37, 35, 33, 31, 29,
27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_u16x32(self, a: mask16x32<Self>, b: u16x32<Self>, c: u16x32<Self>) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask16x32<Avx512>,
b: u16x32<Avx512>,
c: u16x32<Avx512>,
) -> u16x32<Avx512> {
_mm512_mask_blend_epi16(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn min_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
_mm512_min_epu16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_u16x32(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, b: u16x32<Avx512>) -> u16x32<Avx512> {
_mm512_max_epu16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn split_u16x32(self, a: u16x32<Self>) -> (u16x16<Self>, u16x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>) -> (u16x16<Avx512>, u16x16<Avx512>) {
(
_mm512_castsi512_si256(a.into()).simd_into(token),
_mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn load_interleaved_128_u16x32(self, src: &[u16; 32usize]) -> u16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, src: &[u16; 32usize]) -> u16x32<Avx512> {
let lanes: __m512i =
crate::transmute::checked_transmute_copy::<[u16; 32usize], __m512i>(src);
_mm512_permutexvar_epi16(
_mm512_set_epi16(
31, 27, 23, 19, 15, 11, 7, 3, 30, 26, 22, 18, 14, 10, 6, 2, 29, 25, 21, 17,
13, 9, 5, 1, 28, 24, 20, 16, 12, 8, 4, 0,
),
lanes,
)
.simd_into(token)
}
);
kernel(self, src)
}
#[inline(always)]
fn store_interleaved_128_u16x32(self, a: u16x32<Self>, dest: &mut [u16; 32usize]) -> () {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>, dest: &mut [u16; 32usize]) -> () {
let lanes = _mm512_permutexvar_epi16(
_mm512_set_epi16(
31, 23, 15, 7, 30, 22, 14, 6, 29, 21, 13, 5, 28, 20, 12, 4, 27, 19, 11, 3,
26, 18, 10, 2, 25, 17, 9, 1, 24, 16, 8, 0,
),
a.into(),
);
crate::transmute::checked_transmute_store::<__m512i, [u16; 32usize]>(lanes, dest);
}
);
kernel(self, a, dest);
}
#[inline(always)]
fn narrow_u16x32(self, a: u16x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>) -> u8x32<Avx512> {
_mm512_cvtepi16_epi8(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u8_u16x32(self, a: u16x32<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>) -> u8x64<Avx512> {
__m512i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u32_u16x32(self, a: u16x32<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u16x32<Avx512>) -> u32x16<Avx512> {
__m512i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_mask16x32(self, val: bool) -> mask16x32<Self> {
mask16x32 {
val: (if val { 4294967295u64 } else { 0 }) as _,
simd: self,
}
}
#[inline(always)]
fn load_array_mask16x32(self, val: [i16; 32usize]) -> mask16x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: [i16; 32usize]) -> mask16x32<Avx512> {
let lanes = crate::transmute::checked_transmute_copy(&val);
mask16x32 {
val: _mm512_movepi16_mask(lanes),
simd: token,
}
}
);
kernel(self, val)
}
#[inline(always)]
fn as_array_mask16x32(self, a: mask16x32<Self>) -> [i16; 32usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: mask16x32<Avx512>) -> [i16; 32usize] {
let lanes = _mm512_movm_epi16(a.val);
crate::transmute::checked_transmute_copy(&lanes)
}
);
kernel(self, a)
}
#[inline(always)]
fn from_bitmask_mask16x32(self, bits: u64) -> mask16x32<Self> {
mask16x32 {
val: (bits & 4294967295u64) as _,
simd: self,
}
}
#[inline(always)]
fn to_bitmask_mask16x32(self, a: mask16x32<Self>) -> u64 {
u64::from((a).val) & 4294967295u64
}
#[inline(always)]
fn set_mask16x32(self, a: &mut mask16x32<Self>, index: usize, value: bool) -> () {
assert!(
index < 32usize,
"mask lane index {index} is out of bounds for {} lanes",
32usize
);
let bit = 1u64 << index;
let bits = u64::from((a).val);
let bits = if value { bits | bit } else { bits & !bit };
*a = mask16x32 {
val: (bits) as _,
simd: self,
};
}
#[inline(always)]
fn and_mask16x32(self, a: mask16x32<Self>, b: mask16x32<Self>) -> mask16x32<Self> {
mask16x32 {
val: ((u64::from((a).val) & u64::from((b).val)) & 4294967295u64) as _,
simd: self,
}
}
#[inline(always)]
fn or_mask16x32(self, a: mask16x32<Self>, b: mask16x32<Self>) -> mask16x32<Self> {
mask16x32 {
val: ((u64::from((a).val) | u64::from((b).val)) & 4294967295u64) as _,
simd: self,
}
}
#[inline(always)]
fn xor_mask16x32(self, a: mask16x32<Self>, b: mask16x32<Self>) -> mask16x32<Self> {
mask16x32 {
val: ((u64::from((a).val) ^ u64::from((b).val)) & 4294967295u64) as _,
simd: self,
}
}
#[inline(always)]
fn not_mask16x32(self, a: mask16x32<Self>) -> mask16x32<Self> {
mask16x32 {
val: ((!u64::from((a).val)) & 4294967295u64) as _,
simd: self,
}
}
#[inline(always)]
fn select_mask16x32(
self,
a: mask16x32<Self>,
b: mask16x32<Self>,
c: mask16x32<Self>,
) -> mask16x32<Self> {
mask16x32 {
val: (((u64::from((a).val) & u64::from((b).val))
| ((!u64::from((a).val)) & u64::from((c).val)))
& 4294967295u64) as _,
simd: self,
}
}
#[inline(always)]
fn simd_eq_mask16x32(self, a: mask16x32<Self>, b: mask16x32<Self>) -> mask16x32<Self> {
mask16x32 {
val: (!u64::from(a.val ^ b.val) & 4294967295u64) as _,
simd: self,
}
}
#[inline(always)]
fn any_true_mask16x32(self, a: mask16x32<Self>) -> bool {
let bits = u64::from((a).val) & 4294967295u64;
bits != 0
}
#[inline(always)]
fn all_true_mask16x32(self, a: mask16x32<Self>) -> bool {
let bits = u64::from((a).val) & 4294967295u64;
bits == 4294967295u64
}
#[inline(always)]
fn any_false_mask16x32(self, a: mask16x32<Self>) -> bool {
let bits = u64::from((a).val) & 4294967295u64;
bits != 4294967295u64
}
#[inline(always)]
fn all_false_mask16x32(self, a: mask16x32<Self>) -> bool {
let bits = u64::from((a).val) & 4294967295u64;
bits == 0
}
#[inline(always)]
fn split_mask16x32(self, a: mask16x32<Self>) -> (mask16x16<Self>, mask16x16<Self>) {
let bits = u64::from(a.val);
(
mask16x16 {
val: (bits & 65535u64) as _,
simd: self,
},
mask16x16 {
val: ((bits >> 16usize) & 65535u64) as _,
simd: self,
},
)
}
#[inline(always)]
fn splat_i32x16(self, val: i32) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: i32) -> i32x16<Avx512> {
_mm512_set1_epi32(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_i32x16(self, val: [i32; 16usize]) -> i32x16<Self> {
i32x16 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_i32x16(self, val: &[i32; 16usize]) -> i32x16<Self> {
i32x16 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_i32x16(self, a: i32x16<Self>) -> [i32; 16usize] {
crate::transmute::checked_transmute_copy::<__m512i, [i32; 16usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_i32x16(self, a: &i32x16<Self>) -> &[i32; 16usize] {
crate::transmute::checked_cast_ref::<__m512i, [i32; 16usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_i32x16(self, a: &mut i32x16<Self>) -> &mut [i32; 16usize] {
crate::transmute::checked_cast_mut::<__m512i, [i32; 16usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_i32x16(self, a: i32x16<Self>, dest: &mut [i32; 16usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_i32x16(self, a: u8x64<Self>) -> i32x16<Self> {
i32x16 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_i32x16(self, a: i32x16<Self>) -> u8x64<Self> {
u8x64 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_i32x16<const SHIFT: usize>(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i32x16<Avx512>,
b: i32x16<Avx512>,
shift: usize,
) -> i32x16<Avx512> {
if shift >= 16usize {
return b;
}
let idx = _mm512_add_epi8(
_mm512_set_epi8(
63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45,
44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26,
25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
5, 4, 3, 2, 1, 0,
),
_mm512_set1_epi8((shift * 4usize) as i8),
);
let result = _mm512_permutex2var_epi8(
token.cvt_to_bytes_i32x16(a).val.0,
idx,
token.cvt_to_bytes_i32x16(b).val.0,
);
token.cvt_from_bytes_i32x16(u8x64 {
val: crate::support::Aligned512(result),
simd: token,
})
}
);
kernel(self, a, b, SHIFT)
}
#[inline(always)]
fn slide_within_blocks_i32x16<const SHIFT: usize>(
self,
a: i32x16<Self>,
b: i32x16<Self>,
) -> i32x16<Self> {
if SHIFT == 0 {
return a;
}
if SHIFT >= 4usize {
return b;
}
let a = self.cvt_to_bytes_i32x16(a).val.0;
let b = self.cvt_to_bytes_i32x16(b).val.0;
let result = dyn_alignr_512(self, b, a, SHIFT * 4usize);
self.cvt_from_bytes_i32x16(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_within_blocks_i32x16(
self,
a: i32x16<Self>,
indices: u8x64<Self>,
) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, indices: u8x64<Avx512>) -> i32x16<Avx512> {
let result =
_mm512_shuffle_epi8(token.cvt_to_bytes_i32x16(a).val.0, indices.into());
token.cvt_from_bytes_i32x16(u8x64 {
val: crate::support::Aligned512(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
_mm512_add_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
_mm512_sub_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
_mm512_mullo_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
_mm512_and_si512(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
_mm512_or_si512(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
_mm512_xor_si512(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i32x16(self, a: i32x16<Self>) -> i32x16<Self> {
a ^ !0
}
#[inline(always)]
fn shl_i32x16(self, a: i32x16<Self>, shift: u32) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, shift: u32) -> i32x16<Avx512> {
_mm512_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
_mm512_sllv_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_i32x16(self, a: i32x16<Self>, shift: u32) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, shift: u32) -> i32x16<Avx512> {
_mm512_sra_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
_mm512_srav_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> mask32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> mask32x16<Avx512> {
mask32x16 {
val: _mm512_cmpeq_epi32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> mask32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> mask32x16<Avx512> {
mask32x16 {
val: _mm512_cmplt_epi32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> mask32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> mask32x16<Avx512> {
mask32x16 {
val: _mm512_cmple_epi32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> mask32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> mask32x16<Avx512> {
mask32x16 {
val: _mm512_cmpge_epi32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> mask32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> mask32x16<Avx512> {
mask32x16 {
val: _mm512_cmpgt_epi32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
_mm512_permutex2var_epi32(
a.into(),
_mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
_mm512_permutex2var_epi32(
a.into(),
_mm512_setr_epi32(8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
_mm512_permutex2var_epi32(
a.into(),
_mm512_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
_mm512_permutex2var_epi32(
a.into(),
_mm512_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> (i32x16<Self>, i32x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i32x16<Avx512>,
b: i32x16<Avx512>,
) -> (i32x16<Avx512>, i32x16<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm512_permutex2var_epi32(
a,
_mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23),
b,
)
.simd_into(token),
_mm512_permutex2var_epi32(
a,
_mm512_setr_epi32(
8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> (i32x16<Self>, i32x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: i32x16<Avx512>,
b: i32x16<Avx512>,
) -> (i32x16<Avx512>, i32x16<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm512_permutex2var_epi32(
a,
_mm512_setr_epi32(
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30,
),
b,
)
.simd_into(token),
_mm512_permutex2var_epi32(
a,
_mm512_setr_epi32(
1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_i32x16(self, a: mask32x16<Self>, b: i32x16<Self>, c: i32x16<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask32x16<Avx512>,
b: i32x16<Avx512>,
c: i32x16<Avx512>,
) -> i32x16<Avx512> {
_mm512_mask_blend_epi32(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn min_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
_mm512_min_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_i32x16(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>, b: i32x16<Avx512>) -> i32x16<Avx512> {
_mm512_max_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn split_i32x16(self, a: i32x16<Self>) -> (i32x8<Self>, i32x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>) -> (i32x8<Avx512>, i32x8<Avx512>) {
(
_mm512_castsi512_si256(a.into()).simd_into(token),
_mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_i32x16(self, a: i32x16<Self>) -> i32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>) -> i32x16<Avx512> {
_mm512_sub_epi32(_mm512_setzero_si512(), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u8_i32x16(self, a: i32x16<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>) -> u8x64<Avx512> {
__m512i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_u32_i32x16(self, a: i32x16<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>) -> u32x16<Avx512> {
__m512i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_f32_i32x16(self, a: i32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: i32x16<Avx512>) -> f32x16<Avx512> {
_mm512_cvtepi32_ps(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_u32x16(self, val: u32) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: u32) -> u32x16<Avx512> {
_mm512_set1_epi32(val.cast_signed()).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_u32x16(self, val: [u32; 16usize]) -> u32x16<Self> {
u32x16 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_u32x16(self, val: &[u32; 16usize]) -> u32x16<Self> {
u32x16 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_u32x16(self, a: u32x16<Self>) -> [u32; 16usize] {
crate::transmute::checked_transmute_copy::<__m512i, [u32; 16usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_u32x16(self, a: &u32x16<Self>) -> &[u32; 16usize] {
crate::transmute::checked_cast_ref::<__m512i, [u32; 16usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_u32x16(self, a: &mut u32x16<Self>) -> &mut [u32; 16usize] {
crate::transmute::checked_cast_mut::<__m512i, [u32; 16usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_u32x16(self, a: u32x16<Self>, dest: &mut [u32; 16usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_u32x16(self, a: u8x64<Self>) -> u32x16<Self> {
u32x16 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_u32x16(self, a: u32x16<Self>) -> u8x64<Self> {
u8x64 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_u32x16<const SHIFT: usize>(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u32x16<Avx512>,
b: u32x16<Avx512>,
shift: usize,
) -> u32x16<Avx512> {
if shift >= 16usize {
return b;
}
let idx = _mm512_add_epi8(
_mm512_set_epi8(
63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45,
44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26,
25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
5, 4, 3, 2, 1, 0,
),
_mm512_set1_epi8((shift * 4usize) as i8),
);
let result = _mm512_permutex2var_epi8(
token.cvt_to_bytes_u32x16(a).val.0,
idx,
token.cvt_to_bytes_u32x16(b).val.0,
);
token.cvt_from_bytes_u32x16(u8x64 {
val: crate::support::Aligned512(result),
simd: token,
})
}
);
kernel(self, a, b, SHIFT)
}
#[inline(always)]
fn slide_within_blocks_u32x16<const SHIFT: usize>(
self,
a: u32x16<Self>,
b: u32x16<Self>,
) -> u32x16<Self> {
if SHIFT == 0 {
return a;
}
if SHIFT >= 4usize {
return b;
}
let a = self.cvt_to_bytes_u32x16(a).val.0;
let b = self.cvt_to_bytes_u32x16(b).val.0;
let result = dyn_alignr_512(self, b, a, SHIFT * 4usize);
self.cvt_from_bytes_u32x16(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_within_blocks_u32x16(
self,
a: u32x16<Self>,
indices: u8x64<Self>,
) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, indices: u8x64<Avx512>) -> u32x16<Avx512> {
let result =
_mm512_shuffle_epi8(token.cvt_to_bytes_u32x16(a).val.0, indices.into());
token.cvt_from_bytes_u32x16(u8x64 {
val: crate::support::Aligned512(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
_mm512_add_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
_mm512_sub_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
_mm512_mullo_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
_mm512_and_si512(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
_mm512_or_si512(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
_mm512_xor_si512(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u32x16(self, a: u32x16<Self>) -> u32x16<Self> {
a ^ !0
}
#[inline(always)]
fn shl_u32x16(self, a: u32x16<Self>, shift: u32) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, shift: u32) -> u32x16<Avx512> {
_mm512_sll_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
_mm512_sllv_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_u32x16(self, a: u32x16<Self>, shift: u32) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, shift: u32) -> u32x16<Avx512> {
_mm512_srl_epi32(a.into(), _mm_cvtsi32_si128(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
_mm512_srlv_epi32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> mask32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> mask32x16<Avx512> {
mask32x16 {
val: _mm512_cmpeq_epu32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> mask32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> mask32x16<Avx512> {
mask32x16 {
val: _mm512_cmplt_epu32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> mask32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> mask32x16<Avx512> {
mask32x16 {
val: _mm512_cmple_epu32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> mask32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> mask32x16<Avx512> {
mask32x16 {
val: _mm512_cmpge_epu32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> mask32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> mask32x16<Avx512> {
mask32x16 {
val: _mm512_cmpgt_epu32_mask(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
_mm512_permutex2var_epi32(
a.into(),
_mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
_mm512_permutex2var_epi32(
a.into(),
_mm512_setr_epi32(8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
_mm512_permutex2var_epi32(
a.into(),
_mm512_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
_mm512_permutex2var_epi32(
a.into(),
_mm512_setr_epi32(1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> (u32x16<Self>, u32x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u32x16<Avx512>,
b: u32x16<Avx512>,
) -> (u32x16<Avx512>, u32x16<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm512_permutex2var_epi32(
a,
_mm512_setr_epi32(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23),
b,
)
.simd_into(token),
_mm512_permutex2var_epi32(
a,
_mm512_setr_epi32(
8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> (u32x16<Self>, u32x16<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: u32x16<Avx512>,
b: u32x16<Avx512>,
) -> (u32x16<Avx512>, u32x16<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm512_permutex2var_epi32(
a,
_mm512_setr_epi32(
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30,
),
b,
)
.simd_into(token),
_mm512_permutex2var_epi32(
a,
_mm512_setr_epi32(
1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31,
),
b,
)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn select_u32x16(self, a: mask32x16<Self>, b: u32x16<Self>, c: u32x16<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask32x16<Avx512>,
b: u32x16<Avx512>,
c: u32x16<Avx512>,
) -> u32x16<Avx512> {
_mm512_mask_blend_epi32(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn min_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
_mm512_min_epu32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_u32x16(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, b: u32x16<Avx512>) -> u32x16<Avx512> {
_mm512_max_epu32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn split_u32x16(self, a: u32x16<Self>) -> (u32x8<Self>, u32x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>) -> (u32x8<Avx512>, u32x8<Avx512>) {
(
_mm512_castsi512_si256(a.into()).simd_into(token),
_mm512_extracti64x4_epi64::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn load_interleaved_128_u32x16(self, src: &[u32; 16usize]) -> u32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, src: &[u32; 16usize]) -> u32x16<Avx512> {
let lanes: __m512i =
crate::transmute::checked_transmute_copy::<[u32; 16usize], __m512i>(src);
_mm512_permutexvar_epi32(
_mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15),
lanes,
)
.simd_into(token)
}
);
kernel(self, src)
}
#[inline(always)]
fn store_interleaved_128_u32x16(self, a: u32x16<Self>, dest: &mut [u32; 16usize]) -> () {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>, dest: &mut [u32; 16usize]) -> () {
let lanes = _mm512_permutexvar_epi32(
_mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15),
a.into(),
);
crate::transmute::checked_transmute_store::<__m512i, [u32; 16usize]>(lanes, dest);
}
);
kernel(self, a, dest);
}
#[inline(always)]
fn reinterpret_u8_u32x16(self, a: u32x16<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>) -> u8x64<Avx512> {
__m512i::from(a).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_f32_u32x16(self, a: u32x16<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: u32x16<Avx512>) -> f32x16<Avx512> {
_mm512_cvtepu32_ps(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_mask32x16(self, val: bool) -> mask32x16<Self> {
mask32x16 {
val: (if val { 65535u64 } else { 0 }) as _,
simd: self,
}
}
#[inline(always)]
fn load_array_mask32x16(self, val: [i32; 16usize]) -> mask32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: [i32; 16usize]) -> mask32x16<Avx512> {
let lanes = crate::transmute::checked_transmute_copy(&val);
mask32x16 {
val: _mm512_movepi32_mask(lanes),
simd: token,
}
}
);
kernel(self, val)
}
#[inline(always)]
fn as_array_mask32x16(self, a: mask32x16<Self>) -> [i32; 16usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: mask32x16<Avx512>) -> [i32; 16usize] {
let lanes = _mm512_movm_epi32(a.val);
crate::transmute::checked_transmute_copy(&lanes)
}
);
kernel(self, a)
}
#[inline(always)]
fn from_bitmask_mask32x16(self, bits: u64) -> mask32x16<Self> {
mask32x16 {
val: (bits & 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn to_bitmask_mask32x16(self, a: mask32x16<Self>) -> u64 {
u64::from((a).val) & 65535u64
}
#[inline(always)]
fn set_mask32x16(self, a: &mut mask32x16<Self>, index: usize, value: bool) -> () {
assert!(
index < 16usize,
"mask lane index {index} is out of bounds for {} lanes",
16usize
);
let bit = 1u64 << index;
let bits = u64::from((a).val);
let bits = if value { bits | bit } else { bits & !bit };
*a = mask32x16 {
val: (bits) as _,
simd: self,
};
}
#[inline(always)]
fn and_mask32x16(self, a: mask32x16<Self>, b: mask32x16<Self>) -> mask32x16<Self> {
mask32x16 {
val: ((u64::from((a).val) & u64::from((b).val)) & 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn or_mask32x16(self, a: mask32x16<Self>, b: mask32x16<Self>) -> mask32x16<Self> {
mask32x16 {
val: ((u64::from((a).val) | u64::from((b).val)) & 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn xor_mask32x16(self, a: mask32x16<Self>, b: mask32x16<Self>) -> mask32x16<Self> {
mask32x16 {
val: ((u64::from((a).val) ^ u64::from((b).val)) & 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn not_mask32x16(self, a: mask32x16<Self>) -> mask32x16<Self> {
mask32x16 {
val: ((!u64::from((a).val)) & 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn select_mask32x16(
self,
a: mask32x16<Self>,
b: mask32x16<Self>,
c: mask32x16<Self>,
) -> mask32x16<Self> {
mask32x16 {
val: (((u64::from((a).val) & u64::from((b).val))
| ((!u64::from((a).val)) & u64::from((c).val)))
& 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn simd_eq_mask32x16(self, a: mask32x16<Self>, b: mask32x16<Self>) -> mask32x16<Self> {
mask32x16 {
val: (!u64::from(a.val ^ b.val) & 65535u64) as _,
simd: self,
}
}
#[inline(always)]
fn any_true_mask32x16(self, a: mask32x16<Self>) -> bool {
let bits = u64::from((a).val) & 65535u64;
bits != 0
}
#[inline(always)]
fn all_true_mask32x16(self, a: mask32x16<Self>) -> bool {
let bits = u64::from((a).val) & 65535u64;
bits == 65535u64
}
#[inline(always)]
fn any_false_mask32x16(self, a: mask32x16<Self>) -> bool {
let bits = u64::from((a).val) & 65535u64;
bits != 65535u64
}
#[inline(always)]
fn all_false_mask32x16(self, a: mask32x16<Self>) -> bool {
let bits = u64::from((a).val) & 65535u64;
bits == 0
}
#[inline(always)]
fn split_mask32x16(self, a: mask32x16<Self>) -> (mask32x8<Self>, mask32x8<Self>) {
let bits = u64::from(a.val);
(
mask32x8 {
val: (bits & 255u64) as _,
simd: self,
},
mask32x8 {
val: ((bits >> 8usize) & 255u64) as _,
simd: self,
},
)
}
#[inline(always)]
fn splat_f64x8(self, val: f64) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: f64) -> f64x8<Avx512> {
_mm512_set1_pd(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn load_array_f64x8(self, val: [f64; 8usize]) -> f64x8<Self> {
f64x8 {
val: crate::transmute::checked_transmute_copy(&val),
simd: self,
}
}
#[inline(always)]
fn load_array_ref_f64x8(self, val: &[f64; 8usize]) -> f64x8<Self> {
f64x8 {
val: crate::transmute::checked_transmute_copy(val),
simd: self,
}
}
#[inline(always)]
fn as_array_f64x8(self, a: f64x8<Self>) -> [f64; 8usize] {
crate::transmute::checked_transmute_copy::<__m512d, [f64; 8usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_ref_f64x8(self, a: &f64x8<Self>) -> &[f64; 8usize] {
crate::transmute::checked_cast_ref::<__m512d, [f64; 8usize]>(&a.val.0)
}
#[inline(always)]
fn as_array_mut_f64x8(self, a: &mut f64x8<Self>) -> &mut [f64; 8usize] {
crate::transmute::checked_cast_mut::<__m512d, [f64; 8usize]>(&mut a.val.0)
}
#[inline(always)]
fn store_array_f64x8(self, a: f64x8<Self>, dest: &mut [f64; 8usize]) -> () {
crate::transmute::checked_transmute_store(a.val.0, dest);
}
#[inline(always)]
fn cvt_from_bytes_f64x8(self, a: u8x64<Self>) -> f64x8<Self> {
f64x8 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn cvt_to_bytes_f64x8(self, a: f64x8<Self>) -> u8x64<Self> {
u8x64 {
val: crate::transmute::checked_transmute_copy(&a.val),
simd: self,
}
}
#[inline(always)]
fn slide_f64x8<const SHIFT: usize>(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f64x8<Avx512>,
b: f64x8<Avx512>,
shift: usize,
) -> f64x8<Avx512> {
if shift >= 8usize {
return b;
}
let idx = _mm512_add_epi8(
_mm512_set_epi8(
63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45,
44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26,
25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
5, 4, 3, 2, 1, 0,
),
_mm512_set1_epi8((shift * 8usize) as i8),
);
let result = _mm512_permutex2var_epi8(
token.cvt_to_bytes_f64x8(a).val.0,
idx,
token.cvt_to_bytes_f64x8(b).val.0,
);
token.cvt_from_bytes_f64x8(u8x64 {
val: crate::support::Aligned512(result),
simd: token,
})
}
);
kernel(self, a, b, SHIFT)
}
#[inline(always)]
fn slide_within_blocks_f64x8<const SHIFT: usize>(
self,
a: f64x8<Self>,
b: f64x8<Self>,
) -> f64x8<Self> {
if SHIFT == 0 {
return a;
}
if SHIFT >= 2usize {
return b;
}
let a = self.cvt_to_bytes_f64x8(a).val.0;
let b = self.cvt_to_bytes_f64x8(b).val.0;
let result = dyn_alignr_512(self, b, a, SHIFT * 8usize);
self.cvt_from_bytes_f64x8(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_within_blocks_f64x8(self, a: f64x8<Self>, indices: u8x64<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>, indices: u8x64<Avx512>) -> f64x8<Avx512> {
let result = _mm512_shuffle_epi8(token.cvt_to_bytes_f64x8(a).val.0, indices.into());
token.cvt_from_bytes_f64x8(u8x64 {
val: crate::support::Aligned512(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn abs_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>) -> f64x8<Avx512> {
_mm512_andnot_pd(_mm512_set1_pd(-0.0), a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>) -> f64x8<Avx512> {
_mm512_xor_pd(a.into(), _mm512_set1_pd(-0.0)).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn sqrt_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>) -> f64x8<Avx512> {
_mm512_sqrt_pd(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn approximate_recip_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>) -> f64x8<Avx512> {
_mm512_rcp14_pd(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn add_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
_mm512_add_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
_mm512_sub_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
_mm512_mul_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn div_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
_mm512_div_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn copysign_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
let mask = _mm512_set1_pd(-0.0);
_mm512_or_pd(
_mm512_and_pd(mask, b.into()),
_mm512_andnot_pd(mask, a.into()),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> mask64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> mask64x8<Avx512> {
mask64x8 {
val: _mm512_cmp_pd_mask::<0i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> mask64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> mask64x8<Avx512> {
mask64x8 {
val: _mm512_cmp_pd_mask::<17i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> mask64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> mask64x8<Avx512> {
mask64x8 {
val: _mm512_cmp_pd_mask::<18i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_ge_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> mask64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> mask64x8<Avx512> {
mask64x8 {
val: _mm512_cmp_pd_mask::<29i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_gt_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> mask64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> mask64x8<Avx512> {
mask64x8 {
val: _mm512_cmp_pd_mask::<30i32>(a.into(), b.into()),
simd: token,
}
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
_mm512_permutex2var_pd(
a.into(),
_mm512_setr_epi64(0, 8, 1, 9, 2, 10, 3, 11),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
_mm512_permutex2var_pd(
a.into(),
_mm512_setr_epi64(4, 12, 5, 13, 6, 14, 7, 15),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
_mm512_permutex2var_pd(
a.into(),
_mm512_setr_epi64(0, 2, 4, 6, 8, 10, 12, 14),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
_mm512_permutex2var_pd(
a.into(),
_mm512_setr_epi64(1, 3, 5, 7, 9, 11, 13, 15),
b.into(),
)
.simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> (f64x8<Self>, f64x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f64x8<Avx512>,
b: f64x8<Avx512>,
) -> (f64x8<Avx512>, f64x8<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm512_permutex2var_pd(a, _mm512_setr_epi64(0, 8, 1, 9, 2, 10, 3, 11), b)
.simd_into(token),
_mm512_permutex2var_pd(a, _mm512_setr_epi64(4, 12, 5, 13, 6, 14, 7, 15), b)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn deinterleave_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> (f64x8<Self>, f64x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f64x8<Avx512>,
b: f64x8<Avx512>,
) -> (f64x8<Avx512>, f64x8<Avx512>) {
let a = a.into();
let b = b.into();
(
_mm512_permutex2var_pd(a, _mm512_setr_epi64(0, 2, 4, 6, 8, 10, 12, 14), b)
.simd_into(token),
_mm512_permutex2var_pd(a, _mm512_setr_epi64(1, 3, 5, 7, 9, 11, 13, 15), b)
.simd_into(token),
)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
_mm512_max_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
_mm512_min_pd(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_precise_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
_mm512_range_pd::<5i32>(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_precise_f64x8(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>, b: f64x8<Avx512>) -> f64x8<Avx512> {
_mm512_range_pd::<4i32>(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_add_f64x8(self, a: f64x8<Self>, b: f64x8<Self>, c: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f64x8<Avx512>,
b: f64x8<Avx512>,
c: f64x8<Avx512>,
) -> f64x8<Avx512> {
_mm512_fmadd_pd(a.into(), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn mul_sub_f64x8(self, a: f64x8<Self>, b: f64x8<Self>, c: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: f64x8<Avx512>,
b: f64x8<Avx512>,
c: f64x8<Avx512>,
) -> f64x8<Avx512> {
_mm512_fmsub_pd(a.into(), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn floor_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>) -> f64x8<Avx512> {
_mm512_roundscale_pd::<{ _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn ceil_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>) -> f64x8<Avx512> {
_mm512_roundscale_pd::<{ _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn round_ties_even_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>) -> f64x8<Avx512> {
_mm512_roundscale_pd::<{ _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn fract_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
a - self.trunc_f64x8(a)
}
#[inline(always)]
fn trunc_f64x8(self, a: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>) -> f64x8<Avx512> {
_mm512_roundscale_pd::<{ _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC }>(a.into())
.simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn select_f64x8(self, a: mask64x8<Self>, b: f64x8<Self>, c: f64x8<Self>) -> f64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Avx512,
a: mask64x8<Avx512>,
b: f64x8<Avx512>,
c: f64x8<Avx512>,
) -> f64x8<Avx512> {
_mm512_mask_blend_pd(a.val, c.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn split_f64x8(self, a: f64x8<Self>) -> (f64x4<Self>, f64x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>) -> (f64x4<Avx512>, f64x4<Avx512>) {
(
_mm512_castpd512_pd256(a.into()).simd_into(token),
_mm512_extractf64x4_pd::<1>(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn reinterpret_f32_f64x8(self, a: f64x8<Self>) -> f32x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: f64x8<Avx512>) -> f32x16<Avx512> {
_mm512_castpd_ps(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_mask64x8(self, val: bool) -> mask64x8<Self> {
mask64x8 {
val: (if val { 255u64 } else { 0 }) as _,
simd: self,
}
}
#[inline(always)]
fn load_array_mask64x8(self, val: [i64; 8usize]) -> mask64x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, val: [i64; 8usize]) -> mask64x8<Avx512> {
let lanes = crate::transmute::checked_transmute_copy(&val);
mask64x8 {
val: _mm512_movepi64_mask(lanes),
simd: token,
}
}
);
kernel(self, val)
}
#[inline(always)]
fn as_array_mask64x8(self, a: mask64x8<Self>) -> [i64; 8usize] {
crate::kernel!(
#[inline(always)]
fn kernel(token: Avx512, a: mask64x8<Avx512>) -> [i64; 8usize] {
let lanes = _mm512_movm_epi64(a.val);
crate::transmute::checked_transmute_copy(&lanes)
}
);
kernel(self, a)
}
#[inline(always)]
fn from_bitmask_mask64x8(self, bits: u64) -> mask64x8<Self> {
mask64x8 {
val: (bits & 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn to_bitmask_mask64x8(self, a: mask64x8<Self>) -> u64 {
u64::from((a).val) & 255u64
}
#[inline(always)]
fn set_mask64x8(self, a: &mut mask64x8<Self>, index: usize, value: bool) -> () {
assert!(
index < 8usize,
"mask lane index {index} is out of bounds for {} lanes",
8usize
);
let bit = 1u64 << index;
let bits = u64::from((a).val);
let bits = if value { bits | bit } else { bits & !bit };
*a = mask64x8 {
val: (bits) as _,
simd: self,
};
}
#[inline(always)]
fn and_mask64x8(self, a: mask64x8<Self>, b: mask64x8<Self>) -> mask64x8<Self> {
mask64x8 {
val: ((u64::from((a).val) & u64::from((b).val)) & 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn or_mask64x8(self, a: mask64x8<Self>, b: mask64x8<Self>) -> mask64x8<Self> {
mask64x8 {
val: ((u64::from((a).val) | u64::from((b).val)) & 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn xor_mask64x8(self, a: mask64x8<Self>, b: mask64x8<Self>) -> mask64x8<Self> {
mask64x8 {
val: ((u64::from((a).val) ^ u64::from((b).val)) & 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn not_mask64x8(self, a: mask64x8<Self>) -> mask64x8<Self> {
mask64x8 {
val: ((!u64::from((a).val)) & 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn select_mask64x8(
self,
a: mask64x8<Self>,
b: mask64x8<Self>,
c: mask64x8<Self>,
) -> mask64x8<Self> {
mask64x8 {
val: (((u64::from((a).val) & u64::from((b).val))
| ((!u64::from((a).val)) & u64::from((c).val)))
& 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn simd_eq_mask64x8(self, a: mask64x8<Self>, b: mask64x8<Self>) -> mask64x8<Self> {
mask64x8 {
val: (!u64::from(a.val ^ b.val) & 255u64) as _,
simd: self,
}
}
#[inline(always)]
fn any_true_mask64x8(self, a: mask64x8<Self>) -> bool {
let bits = u64::from((a).val) & 255u64;
bits != 0
}
#[inline(always)]
fn all_true_mask64x8(self, a: mask64x8<Self>) -> bool {
let bits = u64::from((a).val) & 255u64;
bits == 255u64
}
#[inline(always)]
fn any_false_mask64x8(self, a: mask64x8<Self>) -> bool {
let bits = u64::from((a).val) & 255u64;
bits != 255u64
}
#[inline(always)]
fn all_false_mask64x8(self, a: mask64x8<Self>) -> bool {
let bits = u64::from((a).val) & 255u64;
bits == 0
}
#[inline(always)]
fn split_mask64x8(self, a: mask64x8<Self>) -> (mask64x4<Self>, mask64x4<Self>) {
let bits = u64::from(a.val);
(
mask64x4 {
val: (bits & 15u64) as _,
simd: self,
},
mask64x4 {
val: ((bits >> 4usize) & 15u64) as _,
simd: self,
},
)
}
}
impl<S: Simd> SimdFrom<__mmask16, S> for mask8x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __mmask16) -> Self {
Self::from_bitmask(simd, u64::from(arch))
}
}
impl<S: Simd> From<mask8x16<S>> for __mmask16 {
#[inline(always)]
#[allow(
trivial_numeric_casts,
reason = "generated uniformly for all __mmask widths"
)]
fn from(value: mask8x16<S>) -> Self {
value.to_bitmask() as __mmask16
}
}
impl<S: Simd> SimdFrom<__mmask8, S> for mask16x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __mmask8) -> Self {
Self::from_bitmask(simd, u64::from(arch))
}
}
impl<S: Simd> From<mask16x8<S>> for __mmask8 {
#[inline(always)]
#[allow(
trivial_numeric_casts,
reason = "generated uniformly for all __mmask widths"
)]
fn from(value: mask16x8<S>) -> Self {
value.to_bitmask() as __mmask8
}
}
impl<S: Simd> SimdFrom<__mmask8, S> for mask32x4<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __mmask8) -> Self {
Self::from_bitmask(simd, u64::from(arch))
}
}
impl<S: Simd> From<mask32x4<S>> for __mmask8 {
#[inline(always)]
#[allow(
trivial_numeric_casts,
reason = "generated uniformly for all __mmask widths"
)]
fn from(value: mask32x4<S>) -> Self {
value.to_bitmask() as __mmask8
}
}
impl<S: Simd> SimdFrom<__mmask8, S> for mask64x2<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __mmask8) -> Self {
Self::from_bitmask(simd, u64::from(arch))
}
}
impl<S: Simd> From<mask64x2<S>> for __mmask8 {
#[inline(always)]
#[allow(
trivial_numeric_casts,
reason = "generated uniformly for all __mmask widths"
)]
fn from(value: mask64x2<S>) -> Self {
value.to_bitmask() as __mmask8
}
}
impl<S: Simd> SimdFrom<__mmask32, S> for mask8x32<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __mmask32) -> Self {
Self::from_bitmask(simd, u64::from(arch))
}
}
impl<S: Simd> From<mask8x32<S>> for __mmask32 {
#[inline(always)]
#[allow(
trivial_numeric_casts,
reason = "generated uniformly for all __mmask widths"
)]
fn from(value: mask8x32<S>) -> Self {
value.to_bitmask() as __mmask32
}
}
impl<S: Simd> SimdFrom<__mmask16, S> for mask16x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __mmask16) -> Self {
Self::from_bitmask(simd, u64::from(arch))
}
}
impl<S: Simd> From<mask16x16<S>> for __mmask16 {
#[inline(always)]
#[allow(
trivial_numeric_casts,
reason = "generated uniformly for all __mmask widths"
)]
fn from(value: mask16x16<S>) -> Self {
value.to_bitmask() as __mmask16
}
}
impl<S: Simd> SimdFrom<__mmask8, S> for mask32x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __mmask8) -> Self {
Self::from_bitmask(simd, u64::from(arch))
}
}
impl<S: Simd> From<mask32x8<S>> for __mmask8 {
#[inline(always)]
#[allow(
trivial_numeric_casts,
reason = "generated uniformly for all __mmask widths"
)]
fn from(value: mask32x8<S>) -> Self {
value.to_bitmask() as __mmask8
}
}
impl<S: Simd> SimdFrom<__mmask8, S> for mask64x4<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __mmask8) -> Self {
Self::from_bitmask(simd, u64::from(arch))
}
}
impl<S: Simd> From<mask64x4<S>> for __mmask8 {
#[inline(always)]
#[allow(
trivial_numeric_casts,
reason = "generated uniformly for all __mmask widths"
)]
fn from(value: mask64x4<S>) -> Self {
value.to_bitmask() as __mmask8
}
}
impl<S: Simd> SimdFrom<__m512, S> for f32x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m512) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<f32x16<S>> for __m512 {
#[inline(always)]
fn from(value: f32x16<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<__m512i, S> for i8x64<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m512i) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i8x64<S>> for __m512i {
#[inline(always)]
fn from(value: i8x64<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<__m512i, S> for u8x64<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m512i) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u8x64<S>> for __m512i {
#[inline(always)]
fn from(value: u8x64<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<__mmask64, S> for mask8x64<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __mmask64) -> Self {
Self::from_bitmask(simd, u64::from(arch))
}
}
impl<S: Simd> From<mask8x64<S>> for __mmask64 {
#[inline(always)]
#[allow(
trivial_numeric_casts,
reason = "generated uniformly for all __mmask widths"
)]
fn from(value: mask8x64<S>) -> Self {
value.to_bitmask() as __mmask64
}
}
impl<S: Simd> SimdFrom<__m512i, S> for i16x32<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m512i) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i16x32<S>> for __m512i {
#[inline(always)]
fn from(value: i16x32<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<__m512i, S> for u16x32<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m512i) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u16x32<S>> for __m512i {
#[inline(always)]
fn from(value: u16x32<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<__mmask32, S> for mask16x32<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __mmask32) -> Self {
Self::from_bitmask(simd, u64::from(arch))
}
}
impl<S: Simd> From<mask16x32<S>> for __mmask32 {
#[inline(always)]
#[allow(
trivial_numeric_casts,
reason = "generated uniformly for all __mmask widths"
)]
fn from(value: mask16x32<S>) -> Self {
value.to_bitmask() as __mmask32
}
}
impl<S: Simd> SimdFrom<__m512i, S> for i32x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m512i) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i32x16<S>> for __m512i {
#[inline(always)]
fn from(value: i32x16<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<__m512i, S> for u32x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m512i) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u32x16<S>> for __m512i {
#[inline(always)]
fn from(value: u32x16<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<__mmask16, S> for mask32x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __mmask16) -> Self {
Self::from_bitmask(simd, u64::from(arch))
}
}
impl<S: Simd> From<mask32x16<S>> for __mmask16 {
#[inline(always)]
#[allow(
trivial_numeric_casts,
reason = "generated uniformly for all __mmask widths"
)]
fn from(value: mask32x16<S>) -> Self {
value.to_bitmask() as __mmask16
}
}
impl<S: Simd> SimdFrom<__m512d, S> for f64x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __m512d) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<f64x8<S>> for __m512d {
#[inline(always)]
fn from(value: f64x8<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<__mmask8, S> for mask64x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: __mmask8) -> Self {
Self::from_bitmask(simd, u64::from(arch))
}
}
impl<S: Simd> From<mask64x8<S>> for __mmask8 {
#[inline(always)]
#[allow(
trivial_numeric_casts,
reason = "generated uniformly for all __mmask widths"
)]
fn from(value: mask64x8<S>) -> Self {
value.to_bitmask() as __mmask8
}
}
crate::kernel!(
#[doc = r" This is a version of the `alignr` intrinsic that takes a non-const shift argument. The shift is still"]
#[doc = r" expected to be constant in practice, so the match statement will be optimized out. This exists because"]
#[doc = r" Rust doesn't currently let you do math on const generics."]
#[inline(always)]
fn dyn_alignr_128(token: Avx512, a: __m128i, b: __m128i, shift: usize) -> __m128i {
match shift {
0usize => _mm_alignr_epi8::<0i32>(a, b),
1usize => _mm_alignr_epi8::<1i32>(a, b),
2usize => _mm_alignr_epi8::<2i32>(a, b),
3usize => _mm_alignr_epi8::<3i32>(a, b),
4usize => _mm_alignr_epi8::<4i32>(a, b),
5usize => _mm_alignr_epi8::<5i32>(a, b),
6usize => _mm_alignr_epi8::<6i32>(a, b),
7usize => _mm_alignr_epi8::<7i32>(a, b),
8usize => _mm_alignr_epi8::<8i32>(a, b),
9usize => _mm_alignr_epi8::<9i32>(a, b),
10usize => _mm_alignr_epi8::<10i32>(a, b),
11usize => _mm_alignr_epi8::<11i32>(a, b),
12usize => _mm_alignr_epi8::<12i32>(a, b),
13usize => _mm_alignr_epi8::<13i32>(a, b),
14usize => _mm_alignr_epi8::<14i32>(a, b),
15usize => _mm_alignr_epi8::<15i32>(a, b),
_ => unreachable!(),
}
}
);
crate::kernel!(
#[doc = r" This is a version of the `alignr` intrinsic that takes a non-const shift argument. The shift is still"]
#[doc = r" expected to be constant in practice, so the match statement will be optimized out. This exists because"]
#[doc = r" Rust doesn't currently let you do math on const generics."]
#[inline(always)]
fn dyn_alignr_256(token: Avx512, a: __m256i, b: __m256i, shift: usize) -> __m256i {
match shift {
0usize => _mm256_alignr_epi8::<0i32>(a, b),
1usize => _mm256_alignr_epi8::<1i32>(a, b),
2usize => _mm256_alignr_epi8::<2i32>(a, b),
3usize => _mm256_alignr_epi8::<3i32>(a, b),
4usize => _mm256_alignr_epi8::<4i32>(a, b),
5usize => _mm256_alignr_epi8::<5i32>(a, b),
6usize => _mm256_alignr_epi8::<6i32>(a, b),
7usize => _mm256_alignr_epi8::<7i32>(a, b),
8usize => _mm256_alignr_epi8::<8i32>(a, b),
9usize => _mm256_alignr_epi8::<9i32>(a, b),
10usize => _mm256_alignr_epi8::<10i32>(a, b),
11usize => _mm256_alignr_epi8::<11i32>(a, b),
12usize => _mm256_alignr_epi8::<12i32>(a, b),
13usize => _mm256_alignr_epi8::<13i32>(a, b),
14usize => _mm256_alignr_epi8::<14i32>(a, b),
15usize => _mm256_alignr_epi8::<15i32>(a, b),
_ => unreachable!(),
}
}
);
crate::kernel!(
#[doc = r" This is a version of the `alignr` intrinsic that takes a non-const shift argument. The shift is still"]
#[doc = r" expected to be constant in practice, so the match statement will be optimized out. This exists because"]
#[doc = r" Rust doesn't currently let you do math on const generics."]
#[inline(always)]
fn dyn_alignr_512(token: Avx512, a: __m512i, b: __m512i, shift: usize) -> __m512i {
match shift {
0usize => _mm512_alignr_epi8::<0i32>(a, b),
1usize => _mm512_alignr_epi8::<1i32>(a, b),
2usize => _mm512_alignr_epi8::<2i32>(a, b),
3usize => _mm512_alignr_epi8::<3i32>(a, b),
4usize => _mm512_alignr_epi8::<4i32>(a, b),
5usize => _mm512_alignr_epi8::<5i32>(a, b),
6usize => _mm512_alignr_epi8::<6i32>(a, b),
7usize => _mm512_alignr_epi8::<7i32>(a, b),
8usize => _mm512_alignr_epi8::<8i32>(a, b),
9usize => _mm512_alignr_epi8::<9i32>(a, b),
10usize => _mm512_alignr_epi8::<10i32>(a, b),
11usize => _mm512_alignr_epi8::<11i32>(a, b),
12usize => _mm512_alignr_epi8::<12i32>(a, b),
13usize => _mm512_alignr_epi8::<13i32>(a, b),
14usize => _mm512_alignr_epi8::<14i32>(a, b),
15usize => _mm512_alignr_epi8::<15i32>(a, b),
_ => unreachable!(),
}
}
);