use crate::{Level, arch_types::ArchTypes, prelude::*, seal::Seal};
use crate::{
f32x4, f32x8, f32x16, f64x2, f64x4, f64x8, i8x16, i8x32, i8x64, i16x8, i16x16, i16x32, i32x4,
i32x8, i32x16, i64x2, i64x4, i64x8, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16,
mask16x32, mask32x4, mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64,
u16x8, u16x16, u16x32, u32x4, u32x8, u32x16, u64x2, u64x4, u64x8,
};
use core::arch::aarch64::*;
#[doc = "A token for Neon intrinsics on aarch64, representing the \"neon\" level."]
#[doc = "# Browsing the documentation"]
#[doc = "The method list on this struct is very verbose."]
#[doc = "A better way to browse the docs is by looking at vector types such as [`u32x4`], [`f32x4`] or [`mask32x4`]."]
#[doc = "They include all the operations listed here, and also provide some additional convenience methods."]
#[derive(Clone, Copy, Debug)]
pub struct Neon {
_private: (),
}
impl Neon {
#[doc = r" Create a SIMD token proving that Neon is available."]
#[doc = r""]
#[doc = r" This function can be called safely from a function with the `neon` target feature"]
#[doc = r" enabled."]
#[doc = r""]
#[doc = r" # Safety"]
#[doc = r""]
#[doc = r" When invoking this function through an `unsafe` block, the caller must ensure that"]
#[doc = r" the current CPU supports `neon`."]
#[inline]
#[target_feature(enable = "neon")]
pub const fn assume_supported() -> Self {
Neon { _private: () }
}
}
impl Seal for Neon {}
impl ArchTypes for Neon {
type f32x4 = crate::support::Aligned128<float32x4_t>;
type i8x16 = crate::support::Aligned128<int8x16_t>;
type u8x16 = crate::support::Aligned128<uint8x16_t>;
type mask8x16 = crate::support::Aligned128<int8x16_t>;
type i16x8 = crate::support::Aligned128<int16x8_t>;
type u16x8 = crate::support::Aligned128<uint16x8_t>;
type mask16x8 = crate::support::Aligned128<int16x8_t>;
type i32x4 = crate::support::Aligned128<int32x4_t>;
type u32x4 = crate::support::Aligned128<uint32x4_t>;
type mask32x4 = crate::support::Aligned128<int32x4_t>;
type f64x2 = crate::support::Aligned128<float64x2_t>;
type i64x2 = crate::support::Aligned128<int64x2_t>;
type u64x2 = crate::support::Aligned128<uint64x2_t>;
type mask64x2 = crate::support::Aligned128<int64x2_t>;
type f32x8 = crate::support::Aligned256<float32x4x2_t>;
type i8x32 = crate::support::Aligned256<int8x16x2_t>;
type u8x32 = crate::support::Aligned256<uint8x16x2_t>;
type mask8x32 = crate::support::Aligned256<int8x16x2_t>;
type i16x16 = crate::support::Aligned256<int16x8x2_t>;
type u16x16 = crate::support::Aligned256<uint16x8x2_t>;
type mask16x16 = crate::support::Aligned256<int16x8x2_t>;
type i32x8 = crate::support::Aligned256<int32x4x2_t>;
type u32x8 = crate::support::Aligned256<uint32x4x2_t>;
type mask32x8 = crate::support::Aligned256<int32x4x2_t>;
type f64x4 = crate::support::Aligned256<float64x2x2_t>;
type i64x4 = crate::support::Aligned256<int64x2x2_t>;
type u64x4 = crate::support::Aligned256<uint64x2x2_t>;
type mask64x4 = crate::support::Aligned256<int64x2x2_t>;
type f32x16 = crate::support::Aligned512<float32x4x4_t>;
type i8x64 = crate::support::Aligned512<int8x16x4_t>;
type u8x64 = crate::support::Aligned512<uint8x16x4_t>;
type mask8x64 = crate::support::Aligned512<int8x16x4_t>;
type i16x32 = crate::support::Aligned512<int16x8x4_t>;
type u16x32 = crate::support::Aligned512<uint16x8x4_t>;
type mask16x32 = crate::support::Aligned512<int16x8x4_t>;
type i32x16 = crate::support::Aligned512<int32x4x4_t>;
type u32x16 = crate::support::Aligned512<uint32x4x4_t>;
type mask32x16 = crate::support::Aligned512<int32x4x4_t>;
type f64x8 = crate::support::Aligned512<float64x2x4_t>;
type i64x8 = crate::support::Aligned512<int64x2x4_t>;
type u64x8 = crate::support::Aligned512<uint64x2x4_t>;
type mask64x8 = crate::support::Aligned512<int64x2x4_t>;
}
impl Simd for Neon {
type f32s = f32x4<Self>;
type f64s = f64x2<Self>;
type u8s = u8x16<Self>;
type i8s = i8x16<Self>;
type u16s = u16x8<Self>;
type i16s = i16x8<Self>;
type u32s = u32x4<Self>;
type i32s = i32x4<Self>;
type u64s = u64x2<Self>;
type i64s = i64x2<Self>;
type mask8s = mask8x16<Self>;
type mask16s = mask16x8<Self>;
type mask32s = mask32x4<Self>;
type mask64s = mask64x2<Self>;
#[inline(always)]
fn level(self) -> Level {
Level::Neon(self)
}
#[inline]
fn vectorize<F: FnOnce() -> R, R>(self, f: F) -> R {
#[target_feature(enable = "neon")]
fn vectorize_neon<F: FnOnce() -> R, R>(f: F) -> R {
f()
}
unsafe { vectorize_neon(f) }
}
#[inline(always)]
fn splat_f32x4(self, val: f32) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, val: f32) -> f32x4<Neon> {
vdupq_n_f32(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_f32x4<const SHIFT: usize>(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = dyn_vext_128(
self,
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn abs_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>) -> f32x4<Neon> {
vabsq_f32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>) -> f32x4<Neon> {
vnegq_f32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn sqrt_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>) -> f32x4<Neon> {
vsqrtq_f32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn approximate_recip_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>) -> f32x4<Neon> {
vrecpeq_f32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn add_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>, b: f32x4<Neon>) -> f32x4<Neon> {
vaddq_f32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>, b: f32x4<Neon>) -> f32x4<Neon> {
vsubq_f32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>, b: f32x4<Neon>) -> f32x4<Neon> {
vmulq_f32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn div_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>, b: f32x4<Neon>) -> f32x4<Neon> {
vdivq_f32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn copysign_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>, b: f32x4<Neon>) -> f32x4<Neon> {
let sign_mask = vdupq_n_u32(1 << 31);
vbslq_f32(sign_mask, b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>, b: f32x4<Neon>) -> f32x4<Neon> {
vmaxq_f32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>, b: f32x4<Neon>) -> f32x4<Neon> {
vminq_f32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>, b: f32x4<Neon>) -> f32x4<Neon> {
vmaxnmq_f32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>, b: f32x4<Neon>) -> f32x4<Neon> {
vminnmq_f32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>, b: f32x4<Neon>) -> mask32x4<Neon> {
vreinterpretq_s32_u32(vceqq_f32(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>, b: f32x4<Neon>) -> mask32x4<Neon> {
vreinterpretq_s32_u32(vcltq_f32(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>, b: f32x4<Neon>) -> mask32x4<Neon> {
vreinterpretq_s32_u32(vcleq_f32(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>, b: f32x4<Neon>) -> f32x4<Neon> {
let x = a.into();
let y = b.into();
vzip1q_f32(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>, b: f32x4<Neon>) -> f32x4<Neon> {
let x = a.into();
let y = b.into();
vzip2q_f32(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>, b: f32x4<Neon>) -> f32x4<Neon> {
let x = a.into();
let y = b.into();
vuzp1q_f32(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>, b: f32x4<Neon>) -> f32x4<Neon> {
let x = a.into();
let y = b.into();
vuzp2q_f32(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> (f32x4<Self>, f32x4<Self>) {
(self.zip_low_f32x4(a, b), self.zip_high_f32x4(a, b))
}
#[inline(always)]
fn deinterleave_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> (f32x4<Self>, f32x4<Self>) {
(self.unzip_low_f32x4(a, b), self.unzip_high_f32x4(a, b))
}
#[inline(always)]
fn mul_add_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>, b: f32x4<Neon>, c: f32x4<Neon>) -> f32x4<Neon> {
vfmaq_f32(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn mul_sub_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>, b: f32x4<Neon>, c: f32x4<Neon>) -> f32x4<Neon> {
vnegq_f32(vfmsq_f32(c.into(), b.into(), a.into())).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn floor_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>) -> f32x4<Neon> {
vrndmq_f32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn ceil_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>) -> f32x4<Neon> {
vrndpq_f32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn round_ties_even_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>) -> f32x4<Neon> {
vrndnq_f32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn fract_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>) -> f32x4<Neon> {
let c1 = vcvtq_s32_f32(a.into());
let c2 = vcvtq_f32_s32(c1);
vsubq_f32(a.into(), c2).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn trunc_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>) -> f32x4<Neon> {
vrndq_f32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn select_f32x4(self, a: mask32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Neon,
a: mask32x4<Neon>,
b: f32x4<Neon>,
c: f32x4<Neon>,
) -> f32x4<Neon> {
vbslq_f32(vreinterpretq_u32_s32(a.into()), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x8<Self> {
f32x8 {
val: crate::support::Aligned256(float32x4x2_t(a.val.0, b.val.0)),
simd: self,
}
}
#[inline(always)]
fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4<Self>; 4usize] {
let native = unsafe { vld4q_f32(src.as_ptr()) };
[
native.0.simd_into(self),
native.1.simd_into(self),
native.2.simd_into(self),
native.3.simd_into(self),
]
}
#[inline(always)]
fn store_four_interleaved_f32x4(
self,
vectors: [f32x4<Self>; 4usize],
dest: &mut [f32; 16usize],
) -> () {
let v0: float32x4_t = vectors[0usize].into();
let v1: float32x4_t = vectors[1usize].into();
let v2: float32x4_t = vectors[2usize].into();
let v3: float32x4_t = vectors[3usize].into();
unsafe {
vst4q_f32(dest.as_mut_ptr(), float32x4x4_t(v0, v1, v2, v3));
}
}
#[inline(always)]
fn widen_f32x4(self, a: f32x4<Self>) -> (f64x2<Self>, f64x2<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>) -> (f64x2<Neon>, f64x2<Neon>) {
(
vcvt_f64_f32(vget_low_f32(a.into())).simd_into(token),
vcvt_high_f64_f32(a.into()).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_u32_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>) -> u32x4<Neon> {
vcvtq_u32_f32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_u32_precise_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
self.cvt_u32_f32x4(a)
}
#[inline(always)]
fn cvt_i32_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f32x4<Neon>) -> i32x4<Neon> {
vcvtq_s32_f32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_i32_precise_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
self.cvt_i32_f32x4(a)
}
#[inline(always)]
fn splat_i8x16(self, val: i8) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, val: i8) -> i8x16<Neon> {
vdupq_n_s8(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_i8x16<const SHIFT: usize>(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = dyn_vext_128(
self,
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>, b: i8x16<Neon>) -> i8x16<Neon> {
vaddq_s8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>, b: i8x16<Neon>) -> i8x16<Neon> {
vsubq_s8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>, b: i8x16<Neon>) -> i8x16<Neon> {
vmulq_s8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>, b: i8x16<Neon>) -> i8x16<Neon> {
vandq_s8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>, b: i8x16<Neon>) -> i8x16<Neon> {
vorrq_s8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>, b: i8x16<Neon>) -> i8x16<Neon> {
veorq_s8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>) -> i8x16<Neon> {
vmvnq_s8(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn shl_i8x16(self, a: i8x16<Self>, shift: u32) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>, shift: u32) -> i8x16<Neon> {
vshlq_s8(a.into(), vdupq_n_s8(shift as i8)).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>, b: i8x16<Neon>) -> i8x16<Neon> {
vshlq_s8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_i8x16(self, a: i8x16<Self>, shift: u32) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>, shift: u32) -> i8x16<Neon> {
vshlq_s8(a.into(), vdupq_n_s8(-(shift as i8))).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>, b: i8x16<Neon>) -> i8x16<Neon> {
vshlq_s8(a.into(), vnegq_s8(b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>, b: i8x16<Neon>) -> i8x16<Neon> {
vmaxq_s8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>, b: i8x16<Neon>) -> i8x16<Neon> {
vminq_s8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>, b: i8x16<Neon>) -> mask8x16<Neon> {
vreinterpretq_s8_u8(vceqq_s8(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>, b: i8x16<Neon>) -> mask8x16<Neon> {
vreinterpretq_s8_u8(vcltq_s8(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>, b: i8x16<Neon>) -> mask8x16<Neon> {
vreinterpretq_s8_u8(vcleq_s8(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>, b: i8x16<Neon>) -> i8x16<Neon> {
let x = a.into();
let y = b.into();
vzip1q_s8(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>, b: i8x16<Neon>) -> i8x16<Neon> {
let x = a.into();
let y = b.into();
vzip2q_s8(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>, b: i8x16<Neon>) -> i8x16<Neon> {
let x = a.into();
let y = b.into();
vuzp1q_s8(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>, b: i8x16<Neon>) -> i8x16<Neon> {
let x = a.into();
let y = b.into();
vuzp2q_s8(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i8x16<Self>, i8x16<Self>) {
(self.zip_low_i8x16(a, b), self.zip_high_i8x16(a, b))
}
#[inline(always)]
fn deinterleave_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i8x16<Self>, i8x16<Self>) {
(self.unzip_low_i8x16(a, b), self.unzip_high_i8x16(a, b))
}
#[inline(always)]
fn select_i8x16(self, a: mask8x16<Self>, b: i8x16<Self>, c: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Neon,
a: mask8x16<Neon>,
b: i8x16<Neon>,
c: i8x16<Neon>,
) -> i8x16<Neon> {
vbslq_s8(vreinterpretq_u8_s8(a.into()), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x32<Self> {
i8x32 {
val: crate::support::Aligned256(int8x16x2_t(a.val.0, b.val.0)),
simd: self,
}
}
#[inline(always)]
fn neg_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>) -> i8x16<Neon> {
vnegq_s8(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16<Self>; 4usize] {
let native = unsafe { vld4q_s8(src.as_ptr()) };
[
native.0.simd_into(self),
native.1.simd_into(self),
native.2.simd_into(self),
native.3.simd_into(self),
]
}
#[inline(always)]
fn store_four_interleaved_i8x16(
self,
vectors: [i8x16<Self>; 4usize],
dest: &mut [i8; 64usize],
) -> () {
let v0: int8x16_t = vectors[0usize].into();
let v1: int8x16_t = vectors[1usize].into();
let v2: int8x16_t = vectors[2usize].into();
let v3: int8x16_t = vectors[3usize].into();
unsafe {
vst4q_s8(dest.as_mut_ptr(), int8x16x4_t(v0, v1, v2, v3));
}
}
#[inline(always)]
fn widen_i8x16(self, a: i8x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i8x16<Neon>) -> (i16x8<Neon>, i16x8<Neon>) {
(
vmovl_s8(vget_low_s8(a.into())).simd_into(token),
vmovl_s8(vget_high_s8(a.into())).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_u8x16(self, val: u8) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, val: u8) -> u8x16<Neon> {
vdupq_n_u8(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_u8x16<const SHIFT: usize>(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = dyn_vext_128(
self,
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, indices: u8x16<Neon>) -> u8x16<Neon> {
let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into());
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn swizzle_dyn_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
self.swizzle_dyn_precise_u8x16(a, indices)
}
#[inline(always)]
fn swizzle_dyn_precise_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, indices: u8x16<Neon>) -> u8x16<Neon> {
let result = vqtbl1q_u8(Bytes::to_bytes(a).val.0, indices.into());
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn add_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, b: u8x16<Neon>) -> u8x16<Neon> {
vaddq_u8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, b: u8x16<Neon>) -> u8x16<Neon> {
vsubq_u8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, b: u8x16<Neon>) -> u8x16<Neon> {
vmulq_u8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, b: u8x16<Neon>) -> u8x16<Neon> {
vandq_u8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, b: u8x16<Neon>) -> u8x16<Neon> {
vorrq_u8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, b: u8x16<Neon>) -> u8x16<Neon> {
veorq_u8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>) -> u8x16<Neon> {
vmvnq_u8(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn shl_u8x16(self, a: u8x16<Self>, shift: u32) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, shift: u32) -> u8x16<Neon> {
vshlq_u8(a.into(), vdupq_n_s8(shift as i8)).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, b: u8x16<Neon>) -> u8x16<Neon> {
vshlq_u8(a.into(), vreinterpretq_s8_u8(b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_u8x16(self, a: u8x16<Self>, shift: u32) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, shift: u32) -> u8x16<Neon> {
vshlq_u8(a.into(), vdupq_n_s8(-(shift as i8))).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, b: u8x16<Neon>) -> u8x16<Neon> {
vshlq_u8(a.into(), vnegq_s8(vreinterpretq_s8_u8(b.into()))).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, b: u8x16<Neon>) -> u8x16<Neon> {
vmaxq_u8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, b: u8x16<Neon>) -> u8x16<Neon> {
vminq_u8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, b: u8x16<Neon>) -> mask8x16<Neon> {
vreinterpretq_s8_u8(vceqq_u8(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, b: u8x16<Neon>) -> mask8x16<Neon> {
vreinterpretq_s8_u8(vcltq_u8(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, b: u8x16<Neon>) -> mask8x16<Neon> {
vreinterpretq_s8_u8(vcleq_u8(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, b: u8x16<Neon>) -> u8x16<Neon> {
let x = a.into();
let y = b.into();
vzip1q_u8(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, b: u8x16<Neon>) -> u8x16<Neon> {
let x = a.into();
let y = b.into();
vzip2q_u8(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, b: u8x16<Neon>) -> u8x16<Neon> {
let x = a.into();
let y = b.into();
vuzp1q_u8(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>, b: u8x16<Neon>) -> u8x16<Neon> {
let x = a.into();
let y = b.into();
vuzp2q_u8(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u8x16<Self>, u8x16<Self>) {
(self.zip_low_u8x16(a, b), self.zip_high_u8x16(a, b))
}
#[inline(always)]
fn deinterleave_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u8x16<Self>, u8x16<Self>) {
(self.unzip_low_u8x16(a, b), self.unzip_high_u8x16(a, b))
}
#[inline(always)]
fn select_u8x16(self, a: mask8x16<Self>, b: u8x16<Self>, c: u8x16<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Neon,
a: mask8x16<Neon>,
b: u8x16<Neon>,
c: u8x16<Neon>,
) -> u8x16<Neon> {
vbslq_u8(vreinterpretq_u8_s8(a.into()), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x32<Self> {
u8x32 {
val: crate::support::Aligned256(uint8x16x2_t(a.val.0, b.val.0)),
simd: self,
}
}
#[inline(always)]
fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16<Self>; 4usize] {
let native = unsafe { vld4q_u8(src.as_ptr()) };
[
native.0.simd_into(self),
native.1.simd_into(self),
native.2.simd_into(self),
native.3.simd_into(self),
]
}
#[inline(always)]
fn store_four_interleaved_u8x16(
self,
vectors: [u8x16<Self>; 4usize],
dest: &mut [u8; 64usize],
) -> () {
let v0: uint8x16_t = vectors[0usize].into();
let v1: uint8x16_t = vectors[1usize].into();
let v2: uint8x16_t = vectors[2usize].into();
let v3: uint8x16_t = vectors[3usize].into();
unsafe {
vst4q_u8(dest.as_mut_ptr(), uint8x16x4_t(v0, v1, v2, v3));
}
}
#[inline(always)]
fn widen_u8x16(self, a: u8x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x16<Neon>) -> (u16x8<Neon>, u16x8<Neon>) {
(
vmovl_u8(vget_low_u8(a.into())).simd_into(token),
vmovl_u8(vget_high_u8(a.into())).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_mask8x16(self, val: bool) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, val: bool) -> mask8x16<Neon> {
let val: i8 = if val { !0 } else { 0 };
vdupq_n_s8(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn from_bitmask_mask8x16(self, bits: u64) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, bits: u64) -> mask8x16<Neon> {
let shifts = crate::transmute::checked_transmute_copy::<[i16; 8], int16x8_t>(&[
15, 14, 13, 12, 11, 10, 9, 8,
]);
let lo = vshlq_u16(vdupq_n_u16(bits as u16), shifts);
let hi = vshlq_u16(vdupq_n_u16((bits >> 8) as u16), shifts);
let lo = vcltq_s16(vreinterpretq_s16_u16(lo), vdupq_n_s16(0));
let hi = vcltq_s16(vreinterpretq_s16_u16(hi), vdupq_n_s16(0));
vcombine_s8(
vmovn_s16(vreinterpretq_s16_u16(lo)),
vmovn_s16(vreinterpretq_s16_u16(hi)),
)
.simd_into(token)
}
);
kernel(self, bits)
}
#[inline(always)]
fn to_bitmask_mask8x16(self, a: mask8x16<Self>) -> u64 {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask8x16<Neon>) -> u64 {
let weights = crate::transmute::checked_transmute_copy::<[u8; 16], uint8x16_t>(&[
1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128,
]);
let bits = vandq_u8(vreinterpretq_u8_s8(a.into()), weights);
let lo = vaddv_u8(vget_low_u8(bits)) as u64;
let hi = vaddv_u8(vget_high_u8(bits)) as u64;
lo | (hi << 8)
}
);
kernel(self, a)
}
#[inline(always)]
fn set_mask8x16(self, a: &mut mask8x16<Self>, index: usize, value: bool) -> () {
assert!(
index < 16usize,
"mask lane index {index} is out of bounds for {} lanes",
16usize
);
let mut lanes: [i8; 16usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn and_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask8x16<Neon>, b: mask8x16<Neon>) -> mask8x16<Neon> {
vandq_s8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask8x16<Neon>, b: mask8x16<Neon>) -> mask8x16<Neon> {
vorrq_s8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask8x16<Neon>, b: mask8x16<Neon>) -> mask8x16<Neon> {
veorq_s8(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_mask8x16(self, a: mask8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask8x16<Neon>) -> mask8x16<Neon> {
vmvnq_s8(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn select_mask8x16(
self,
a: mask8x16<Self>,
b: mask8x16<Self>,
c: mask8x16<Self>,
) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Neon,
a: mask8x16<Neon>,
b: mask8x16<Neon>,
c: mask8x16<Neon>,
) -> mask8x16<Neon> {
vbslq_s8(vreinterpretq_u8_s8(a.into()), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn simd_eq_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask8x16<Neon>, b: mask8x16<Neon>) -> mask8x16<Neon> {
vreinterpretq_s8_u8(vceqq_s8(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn any_true_mask8x16(self, a: mask8x16<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask8x16<Neon>) -> bool {
vmaxvq_u32(vreinterpretq_u32_s8(a.into())) != 0
}
);
kernel(self, a)
}
#[inline(always)]
fn all_true_mask8x16(self, a: mask8x16<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask8x16<Neon>) -> bool {
vminvq_u32(vreinterpretq_u32_s8(a.into())) == 0xffffffff
}
);
kernel(self, a)
}
#[inline(always)]
fn any_false_mask8x16(self, a: mask8x16<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask8x16<Neon>) -> bool {
vminvq_u32(vreinterpretq_u32_s8(a.into())) != 0xffffffff
}
);
kernel(self, a)
}
#[inline(always)]
fn all_false_mask8x16(self, a: mask8x16<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask8x16<Neon>) -> bool {
vmaxvq_u32(vreinterpretq_u32_s8(a.into())) == 0
}
);
kernel(self, a)
}
#[inline(always)]
fn combine_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x32<Self> {
mask8x32 {
val: crate::support::Aligned256(int8x16x2_t(a.val.0, b.val.0)),
simd: self,
}
}
#[inline(always)]
fn splat_i16x8(self, val: i16) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, val: i16) -> i16x8<Neon> {
vdupq_n_s16(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_i16x8<const SHIFT: usize>(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = dyn_vext_128(
self,
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 2usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, b: i16x8<Neon>) -> i16x8<Neon> {
vaddq_s16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, b: i16x8<Neon>) -> i16x8<Neon> {
vsubq_s16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, b: i16x8<Neon>) -> i16x8<Neon> {
vmulq_s16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, b: i16x8<Neon>) -> i16x8<Neon> {
vandq_s16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, b: i16x8<Neon>) -> i16x8<Neon> {
vorrq_s16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, b: i16x8<Neon>) -> i16x8<Neon> {
veorq_s16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>) -> i16x8<Neon> {
vmvnq_s16(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn shl_i16x8(self, a: i16x8<Self>, shift: u32) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, shift: u32) -> i16x8<Neon> {
vshlq_s16(a.into(), vdupq_n_s16(shift as i16)).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, b: i16x8<Neon>) -> i16x8<Neon> {
vshlq_s16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_i16x8(self, a: i16x8<Self>, shift: u32) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, shift: u32) -> i16x8<Neon> {
vshlq_s16(a.into(), vdupq_n_s16(-(shift as i16))).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, b: i16x8<Neon>) -> i16x8<Neon> {
vshlq_s16(a.into(), vnegq_s16(b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, b: i16x8<Neon>) -> i16x8<Neon> {
vmaxq_s16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, b: i16x8<Neon>) -> i16x8<Neon> {
vminq_s16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, b: i16x8<Neon>) -> mask16x8<Neon> {
vreinterpretq_s16_u16(vceqq_s16(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, b: i16x8<Neon>) -> mask16x8<Neon> {
vreinterpretq_s16_u16(vcltq_s16(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, b: i16x8<Neon>) -> mask16x8<Neon> {
vreinterpretq_s16_u16(vcleq_s16(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, b: i16x8<Neon>) -> i16x8<Neon> {
let x = a.into();
let y = b.into();
vzip1q_s16(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, b: i16x8<Neon>) -> i16x8<Neon> {
let x = a.into();
let y = b.into();
vzip2q_s16(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, b: i16x8<Neon>) -> i16x8<Neon> {
let x = a.into();
let y = b.into();
vuzp1q_s16(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, b: i16x8<Neon>) -> i16x8<Neon> {
let x = a.into();
let y = b.into();
vuzp2q_s16(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i16x8<Self>, i16x8<Self>) {
(self.zip_low_i16x8(a, b), self.zip_high_i16x8(a, b))
}
#[inline(always)]
fn deinterleave_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i16x8<Self>, i16x8<Self>) {
(self.unzip_low_i16x8(a, b), self.unzip_high_i16x8(a, b))
}
#[inline(always)]
fn select_i16x8(self, a: mask16x8<Self>, b: i16x8<Self>, c: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Neon,
a: mask16x8<Neon>,
b: i16x8<Neon>,
c: i16x8<Neon>,
) -> i16x8<Neon> {
vbslq_s16(vreinterpretq_u16_s16(a.into()), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x16<Self> {
i16x16 {
val: crate::support::Aligned256(int16x8x2_t(a.val.0, b.val.0)),
simd: self,
}
}
#[inline(always)]
fn neg_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>) -> i16x8<Neon> {
vnegq_s16(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8<Self>; 4usize] {
let native = unsafe { vld4q_s16(src.as_ptr()) };
[
native.0.simd_into(self),
native.1.simd_into(self),
native.2.simd_into(self),
native.3.simd_into(self),
]
}
#[inline(always)]
fn store_four_interleaved_i16x8(
self,
vectors: [i16x8<Self>; 4usize],
dest: &mut [i16; 32usize],
) -> () {
let v0: int16x8_t = vectors[0usize].into();
let v1: int16x8_t = vectors[1usize].into();
let v2: int16x8_t = vectors[2usize].into();
let v3: int16x8_t = vectors[3usize].into();
unsafe {
vst4q_s16(dest.as_mut_ptr(), int16x8x4_t(v0, v1, v2, v3));
}
}
#[inline(always)]
fn widen_i16x8(self, a: i16x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>) -> (i32x4<Neon>, i32x4<Neon>) {
(
vmovl_s16(vget_low_s16(a.into())).simd_into(token),
vmovl_s16(vget_high_s16(a.into())).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, b: i16x8<Neon>) -> i8x16<Neon> {
vcombine_s8(vmovn_s16(a.into()), vmovn_s16(b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i16x8<Neon>, b: i16x8<Neon>) -> i8x16<Neon> {
vcombine_s8(vqmovn_s16(a.into()), vqmovn_s16(b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn relaxed_narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= i8::MIN as i16 && value <= i8::MAX as i16 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.narrow_i16x8(a, b)
}
#[inline(always)]
fn splat_u16x8(self, val: u16) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, val: u16) -> u16x8<Neon> {
vdupq_n_u16(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_u16x8<const SHIFT: usize>(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = dyn_vext_128(
self,
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 2usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, b: u16x8<Neon>) -> u16x8<Neon> {
vaddq_u16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, b: u16x8<Neon>) -> u16x8<Neon> {
vsubq_u16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, b: u16x8<Neon>) -> u16x8<Neon> {
vmulq_u16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, b: u16x8<Neon>) -> u16x8<Neon> {
vandq_u16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, b: u16x8<Neon>) -> u16x8<Neon> {
vorrq_u16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, b: u16x8<Neon>) -> u16x8<Neon> {
veorq_u16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>) -> u16x8<Neon> {
vmvnq_u16(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn shl_u16x8(self, a: u16x8<Self>, shift: u32) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, shift: u32) -> u16x8<Neon> {
vshlq_u16(a.into(), vdupq_n_s16(shift as i16)).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, b: u16x8<Neon>) -> u16x8<Neon> {
vshlq_u16(a.into(), vreinterpretq_s16_u16(b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_u16x8(self, a: u16x8<Self>, shift: u32) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, shift: u32) -> u16x8<Neon> {
vshlq_u16(a.into(), vdupq_n_s16(-(shift as i16))).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, b: u16x8<Neon>) -> u16x8<Neon> {
vshlq_u16(a.into(), vnegq_s16(vreinterpretq_s16_u16(b.into()))).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, b: u16x8<Neon>) -> u16x8<Neon> {
vmaxq_u16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, b: u16x8<Neon>) -> u16x8<Neon> {
vminq_u16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, b: u16x8<Neon>) -> mask16x8<Neon> {
vreinterpretq_s16_u16(vceqq_u16(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, b: u16x8<Neon>) -> mask16x8<Neon> {
vreinterpretq_s16_u16(vcltq_u16(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, b: u16x8<Neon>) -> mask16x8<Neon> {
vreinterpretq_s16_u16(vcleq_u16(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, b: u16x8<Neon>) -> u16x8<Neon> {
let x = a.into();
let y = b.into();
vzip1q_u16(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, b: u16x8<Neon>) -> u16x8<Neon> {
let x = a.into();
let y = b.into();
vzip2q_u16(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, b: u16x8<Neon>) -> u16x8<Neon> {
let x = a.into();
let y = b.into();
vuzp1q_u16(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, b: u16x8<Neon>) -> u16x8<Neon> {
let x = a.into();
let y = b.into();
vuzp2q_u16(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u16x8<Self>, u16x8<Self>) {
(self.zip_low_u16x8(a, b), self.zip_high_u16x8(a, b))
}
#[inline(always)]
fn deinterleave_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u16x8<Self>, u16x8<Self>) {
(self.unzip_low_u16x8(a, b), self.unzip_high_u16x8(a, b))
}
#[inline(always)]
fn select_u16x8(self, a: mask16x8<Self>, b: u16x8<Self>, c: u16x8<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Neon,
a: mask16x8<Neon>,
b: u16x8<Neon>,
c: u16x8<Neon>,
) -> u16x8<Neon> {
vbslq_u16(vreinterpretq_u16_s16(a.into()), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x16<Self> {
u16x16 {
val: crate::support::Aligned256(uint16x8x2_t(a.val.0, b.val.0)),
simd: self,
}
}
#[inline(always)]
fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8<Self>; 4usize] {
let native = unsafe { vld4q_u16(src.as_ptr()) };
[
native.0.simd_into(self),
native.1.simd_into(self),
native.2.simd_into(self),
native.3.simd_into(self),
]
}
#[inline(always)]
fn store_four_interleaved_u16x8(
self,
vectors: [u16x8<Self>; 4usize],
dest: &mut [u16; 32usize],
) -> () {
let v0: uint16x8_t = vectors[0usize].into();
let v1: uint16x8_t = vectors[1usize].into();
let v2: uint16x8_t = vectors[2usize].into();
let v3: uint16x8_t = vectors[3usize].into();
unsafe {
vst4q_u16(dest.as_mut_ptr(), uint16x8x4_t(v0, v1, v2, v3));
}
}
#[inline(always)]
fn widen_u16x8(self, a: u16x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>) -> (u32x4<Neon>, u32x4<Neon>) {
(
vmovl_u16(vget_low_u16(a.into())).simd_into(token),
vmovl_u16(vget_high_u16(a.into())).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, b: u16x8<Neon>) -> u8x16<Neon> {
vcombine_u8(vmovn_u16(a.into()), vmovn_u16(b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u16x8<Neon>, b: u16x8<Neon>) -> u8x16<Neon> {
vcombine_u8(vqmovn_u16(a.into()), vqmovn_u16(b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn relaxed_narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= u8::MIN as u16 && value <= u8::MAX as u16 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.narrow_u16x8(a, b)
}
#[inline(always)]
fn splat_mask16x8(self, val: bool) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, val: bool) -> mask16x8<Neon> {
let val: i16 = if val { !0 } else { 0 };
vdupq_n_s16(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn from_bitmask_mask16x8(self, bits: u64) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, bits: u64) -> mask16x8<Neon> {
let shifts = crate::transmute::checked_transmute_copy::<[i16; 8], int16x8_t>(&[
15, 14, 13, 12, 11, 10, 9, 8,
]);
let shifted = vshlq_u16(vdupq_n_u16(bits as u16), shifts);
let mask = vcltq_s16(vreinterpretq_s16_u16(shifted), vdupq_n_s16(0));
vreinterpretq_s16_u16(mask).simd_into(token)
}
);
kernel(self, bits)
}
#[inline(always)]
fn to_bitmask_mask16x8(self, a: mask16x8<Self>) -> u64 {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask16x8<Neon>) -> u64 {
let weights = crate::transmute::checked_transmute_copy::<[u16; 8], uint16x8_t>(&[
1, 2, 4, 8, 16, 32, 64, 128,
]);
let bits = vandq_u16(vreinterpretq_u16_s16(a.into()), weights);
vaddvq_u16(bits) as u64
}
);
kernel(self, a)
}
#[inline(always)]
fn set_mask16x8(self, a: &mut mask16x8<Self>, index: usize, value: bool) -> () {
assert!(
index < 8usize,
"mask lane index {index} is out of bounds for {} lanes",
8usize
);
let mut lanes: [i16; 8usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn and_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask16x8<Neon>, b: mask16x8<Neon>) -> mask16x8<Neon> {
vandq_s16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask16x8<Neon>, b: mask16x8<Neon>) -> mask16x8<Neon> {
vorrq_s16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask16x8<Neon>, b: mask16x8<Neon>) -> mask16x8<Neon> {
veorq_s16(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_mask16x8(self, a: mask16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask16x8<Neon>) -> mask16x8<Neon> {
vmvnq_s16(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn select_mask16x8(
self,
a: mask16x8<Self>,
b: mask16x8<Self>,
c: mask16x8<Self>,
) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Neon,
a: mask16x8<Neon>,
b: mask16x8<Neon>,
c: mask16x8<Neon>,
) -> mask16x8<Neon> {
vbslq_s16(vreinterpretq_u16_s16(a.into()), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn simd_eq_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask16x8<Neon>, b: mask16x8<Neon>) -> mask16x8<Neon> {
vreinterpretq_s16_u16(vceqq_s16(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn any_true_mask16x8(self, a: mask16x8<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask16x8<Neon>) -> bool {
vmaxvq_u32(vreinterpretq_u32_s16(a.into())) != 0
}
);
kernel(self, a)
}
#[inline(always)]
fn all_true_mask16x8(self, a: mask16x8<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask16x8<Neon>) -> bool {
vminvq_u32(vreinterpretq_u32_s16(a.into())) == 0xffffffff
}
);
kernel(self, a)
}
#[inline(always)]
fn any_false_mask16x8(self, a: mask16x8<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask16x8<Neon>) -> bool {
vminvq_u32(vreinterpretq_u32_s16(a.into())) != 0xffffffff
}
);
kernel(self, a)
}
#[inline(always)]
fn all_false_mask16x8(self, a: mask16x8<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask16x8<Neon>) -> bool {
vmaxvq_u32(vreinterpretq_u32_s16(a.into())) == 0
}
);
kernel(self, a)
}
#[inline(always)]
fn combine_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x16<Self> {
mask16x16 {
val: crate::support::Aligned256(int16x8x2_t(a.val.0, b.val.0)),
simd: self,
}
}
#[inline(always)]
fn splat_i32x4(self, val: i32) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, val: i32) -> i32x4<Neon> {
vdupq_n_s32(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_i32x4<const SHIFT: usize>(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = dyn_vext_128(
self,
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, b: i32x4<Neon>) -> i32x4<Neon> {
vaddq_s32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, b: i32x4<Neon>) -> i32x4<Neon> {
vsubq_s32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, b: i32x4<Neon>) -> i32x4<Neon> {
vmulq_s32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, b: i32x4<Neon>) -> i32x4<Neon> {
vandq_s32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, b: i32x4<Neon>) -> i32x4<Neon> {
vorrq_s32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, b: i32x4<Neon>) -> i32x4<Neon> {
veorq_s32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>) -> i32x4<Neon> {
vmvnq_s32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn shl_i32x4(self, a: i32x4<Self>, shift: u32) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, shift: u32) -> i32x4<Neon> {
vshlq_s32(a.into(), vdupq_n_s32(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, b: i32x4<Neon>) -> i32x4<Neon> {
vshlq_s32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_i32x4(self, a: i32x4<Self>, shift: u32) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, shift: u32) -> i32x4<Neon> {
vshlq_s32(a.into(), vdupq_n_s32(-shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, b: i32x4<Neon>) -> i32x4<Neon> {
vshlq_s32(a.into(), vnegq_s32(b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, b: i32x4<Neon>) -> i32x4<Neon> {
vmaxq_s32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, b: i32x4<Neon>) -> i32x4<Neon> {
vminq_s32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, b: i32x4<Neon>) -> mask32x4<Neon> {
vreinterpretq_s32_u32(vceqq_s32(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, b: i32x4<Neon>) -> mask32x4<Neon> {
vreinterpretq_s32_u32(vcltq_s32(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, b: i32x4<Neon>) -> mask32x4<Neon> {
vreinterpretq_s32_u32(vcleq_s32(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, b: i32x4<Neon>) -> i32x4<Neon> {
let x = a.into();
let y = b.into();
vzip1q_s32(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, b: i32x4<Neon>) -> i32x4<Neon> {
let x = a.into();
let y = b.into();
vzip2q_s32(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, b: i32x4<Neon>) -> i32x4<Neon> {
let x = a.into();
let y = b.into();
vuzp1q_s32(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, b: i32x4<Neon>) -> i32x4<Neon> {
let x = a.into();
let y = b.into();
vuzp2q_s32(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i32x4<Self>, i32x4<Self>) {
(self.zip_low_i32x4(a, b), self.zip_high_i32x4(a, b))
}
#[inline(always)]
fn deinterleave_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i32x4<Self>, i32x4<Self>) {
(self.unzip_low_i32x4(a, b), self.unzip_high_i32x4(a, b))
}
#[inline(always)]
fn select_i32x4(self, a: mask32x4<Self>, b: i32x4<Self>, c: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Neon,
a: mask32x4<Neon>,
b: i32x4<Neon>,
c: i32x4<Neon>,
) -> i32x4<Neon> {
vbslq_s32(vreinterpretq_u32_s32(a.into()), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x8<Self> {
i32x8 {
val: crate::support::Aligned256(int32x4x2_t(a.val.0, b.val.0)),
simd: self,
}
}
#[inline(always)]
fn neg_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>) -> i32x4<Neon> {
vnegq_s32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4<Self>; 4usize] {
let native = unsafe { vld4q_s32(src.as_ptr()) };
[
native.0.simd_into(self),
native.1.simd_into(self),
native.2.simd_into(self),
native.3.simd_into(self),
]
}
#[inline(always)]
fn store_four_interleaved_i32x4(
self,
vectors: [i32x4<Self>; 4usize],
dest: &mut [i32; 16usize],
) -> () {
let v0: int32x4_t = vectors[0usize].into();
let v1: int32x4_t = vectors[1usize].into();
let v2: int32x4_t = vectors[2usize].into();
let v3: int32x4_t = vectors[3usize].into();
unsafe {
vst4q_s32(dest.as_mut_ptr(), int32x4x4_t(v0, v1, v2, v3));
}
}
#[inline(always)]
fn widen_i32x4(self, a: i32x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>) -> (i64x2<Neon>, i64x2<Neon>) {
(
vmovl_s32(vget_low_s32(a.into())).simd_into(token),
vmovl_s32(vget_high_s32(a.into())).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, b: i32x4<Neon>) -> i16x8<Neon> {
vcombine_s16(vmovn_s32(a.into()), vmovn_s32(b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>, b: i32x4<Neon>) -> i16x8<Neon> {
vcombine_s16(vqmovn_s32(a.into()), vqmovn_s32(b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn relaxed_narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= i16::MIN as i32 && value <= i16::MAX as i32 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.narrow_i32x4(a, b)
}
#[inline(always)]
fn cvt_f32_i32x4(self, a: i32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i32x4<Neon>) -> f32x4<Neon> {
vcvtq_f32_s32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_u32x4(self, val: u32) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, val: u32) -> u32x4<Neon> {
vdupq_n_u32(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_u32x4<const SHIFT: usize>(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = dyn_vext_128(
self,
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, b: u32x4<Neon>) -> u32x4<Neon> {
vaddq_u32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, b: u32x4<Neon>) -> u32x4<Neon> {
vsubq_u32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, b: u32x4<Neon>) -> u32x4<Neon> {
vmulq_u32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn and_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, b: u32x4<Neon>) -> u32x4<Neon> {
vandq_u32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, b: u32x4<Neon>) -> u32x4<Neon> {
vorrq_u32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, b: u32x4<Neon>) -> u32x4<Neon> {
veorq_u32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>) -> u32x4<Neon> {
vmvnq_u32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn shl_u32x4(self, a: u32x4<Self>, shift: u32) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, shift: u32) -> u32x4<Neon> {
vshlq_u32(a.into(), vdupq_n_s32(shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, b: u32x4<Neon>) -> u32x4<Neon> {
vshlq_u32(a.into(), vreinterpretq_s32_u32(b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_u32x4(self, a: u32x4<Self>, shift: u32) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, shift: u32) -> u32x4<Neon> {
vshlq_u32(a.into(), vdupq_n_s32(-shift.cast_signed())).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, b: u32x4<Neon>) -> u32x4<Neon> {
vshlq_u32(a.into(), vnegq_s32(vreinterpretq_s32_u32(b.into()))).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, b: u32x4<Neon>) -> u32x4<Neon> {
vmaxq_u32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, b: u32x4<Neon>) -> u32x4<Neon> {
vminq_u32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, b: u32x4<Neon>) -> mask32x4<Neon> {
vreinterpretq_s32_u32(vceqq_u32(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, b: u32x4<Neon>) -> mask32x4<Neon> {
vreinterpretq_s32_u32(vcltq_u32(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, b: u32x4<Neon>) -> mask32x4<Neon> {
vreinterpretq_s32_u32(vcleq_u32(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, b: u32x4<Neon>) -> u32x4<Neon> {
let x = a.into();
let y = b.into();
vzip1q_u32(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, b: u32x4<Neon>) -> u32x4<Neon> {
let x = a.into();
let y = b.into();
vzip2q_u32(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, b: u32x4<Neon>) -> u32x4<Neon> {
let x = a.into();
let y = b.into();
vuzp1q_u32(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, b: u32x4<Neon>) -> u32x4<Neon> {
let x = a.into();
let y = b.into();
vuzp2q_u32(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u32x4<Self>, u32x4<Self>) {
(self.zip_low_u32x4(a, b), self.zip_high_u32x4(a, b))
}
#[inline(always)]
fn deinterleave_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u32x4<Self>, u32x4<Self>) {
(self.unzip_low_u32x4(a, b), self.unzip_high_u32x4(a, b))
}
#[inline(always)]
fn select_u32x4(self, a: mask32x4<Self>, b: u32x4<Self>, c: u32x4<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Neon,
a: mask32x4<Neon>,
b: u32x4<Neon>,
c: u32x4<Neon>,
) -> u32x4<Neon> {
vbslq_u32(vreinterpretq_u32_s32(a.into()), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x8<Self> {
u32x8 {
val: crate::support::Aligned256(uint32x4x2_t(a.val.0, b.val.0)),
simd: self,
}
}
#[inline(always)]
fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4<Self>; 4usize] {
let native = unsafe { vld4q_u32(src.as_ptr()) };
[
native.0.simd_into(self),
native.1.simd_into(self),
native.2.simd_into(self),
native.3.simd_into(self),
]
}
#[inline(always)]
fn store_four_interleaved_u32x4(
self,
vectors: [u32x4<Self>; 4usize],
dest: &mut [u32; 16usize],
) -> () {
let v0: uint32x4_t = vectors[0usize].into();
let v1: uint32x4_t = vectors[1usize].into();
let v2: uint32x4_t = vectors[2usize].into();
let v3: uint32x4_t = vectors[3usize].into();
unsafe {
vst4q_u32(dest.as_mut_ptr(), uint32x4x4_t(v0, v1, v2, v3));
}
}
#[inline(always)]
fn widen_u32x4(self, a: u32x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>) -> (u64x2<Neon>, u64x2<Neon>) {
(
vmovl_u32(vget_low_u32(a.into())).simd_into(token),
vmovl_u32(vget_high_u32(a.into())).simd_into(token),
)
}
);
kernel(self, a)
}
#[inline(always)]
fn narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, b: u32x4<Neon>) -> u16x8<Neon> {
vcombine_u16(vmovn_u32(a.into()), vmovn_u32(b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>, b: u32x4<Neon>) -> u16x8<Neon> {
vcombine_u16(vqmovn_u32(a.into()), vqmovn_u32(b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn relaxed_narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= u16::MIN as u32 && value <= u16::MAX as u32 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.narrow_u32x4(a, b)
}
#[inline(always)]
fn cvt_f32_u32x4(self, a: u32x4<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u32x4<Neon>) -> f32x4<Neon> {
vcvtq_f32_u32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_mask32x4(self, val: bool) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, val: bool) -> mask32x4<Neon> {
let val: i32 = if val { !0 } else { 0 };
vdupq_n_s32(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn from_bitmask_mask32x4(self, bits: u64) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, bits: u64) -> mask32x4<Neon> {
let shifts = crate::transmute::checked_transmute_copy::<[i32; 4], int32x4_t>(&[
31, 30, 29, 28,
]);
let shifted = vshlq_u32(vdupq_n_u32(bits as u32), shifts);
let mask = vcltq_s32(vreinterpretq_s32_u32(shifted), vdupq_n_s32(0));
vreinterpretq_s32_u32(mask).simd_into(token)
}
);
kernel(self, bits)
}
#[inline(always)]
fn to_bitmask_mask32x4(self, a: mask32x4<Self>) -> u64 {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask32x4<Neon>) -> u64 {
let weights =
crate::transmute::checked_transmute_copy::<[u32; 4], uint32x4_t>(&[1, 2, 4, 8]);
let bits = vandq_u32(vreinterpretq_u32_s32(a.into()), weights);
vaddvq_u32(bits) as u64
}
);
kernel(self, a)
}
#[inline(always)]
fn set_mask32x4(self, a: &mut mask32x4<Self>, index: usize, value: bool) -> () {
assert!(
index < 4usize,
"mask lane index {index} is out of bounds for {} lanes",
4usize
);
let mut lanes: [i32; 4usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn and_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask32x4<Neon>, b: mask32x4<Neon>) -> mask32x4<Neon> {
vandq_s32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask32x4<Neon>, b: mask32x4<Neon>) -> mask32x4<Neon> {
vorrq_s32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask32x4<Neon>, b: mask32x4<Neon>) -> mask32x4<Neon> {
veorq_s32(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_mask32x4(self, a: mask32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask32x4<Neon>) -> mask32x4<Neon> {
vmvnq_s32(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn select_mask32x4(
self,
a: mask32x4<Self>,
b: mask32x4<Self>,
c: mask32x4<Self>,
) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Neon,
a: mask32x4<Neon>,
b: mask32x4<Neon>,
c: mask32x4<Neon>,
) -> mask32x4<Neon> {
vbslq_s32(vreinterpretq_u32_s32(a.into()), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn simd_eq_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask32x4<Neon>, b: mask32x4<Neon>) -> mask32x4<Neon> {
vreinterpretq_s32_u32(vceqq_s32(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn any_true_mask32x4(self, a: mask32x4<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask32x4<Neon>) -> bool {
vmaxvq_u32(vreinterpretq_u32_s32(a.into())) != 0
}
);
kernel(self, a)
}
#[inline(always)]
fn all_true_mask32x4(self, a: mask32x4<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask32x4<Neon>) -> bool {
vminvq_u32(vreinterpretq_u32_s32(a.into())) == 0xffffffff
}
);
kernel(self, a)
}
#[inline(always)]
fn any_false_mask32x4(self, a: mask32x4<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask32x4<Neon>) -> bool {
vminvq_u32(vreinterpretq_u32_s32(a.into())) != 0xffffffff
}
);
kernel(self, a)
}
#[inline(always)]
fn all_false_mask32x4(self, a: mask32x4<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask32x4<Neon>) -> bool {
vmaxvq_u32(vreinterpretq_u32_s32(a.into())) == 0
}
);
kernel(self, a)
}
#[inline(always)]
fn combine_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x8<Self> {
mask32x8 {
val: crate::support::Aligned256(int32x4x2_t(a.val.0, b.val.0)),
simd: self,
}
}
#[inline(always)]
fn splat_f64x2(self, val: f64) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, val: f64) -> f64x2<Neon> {
vdupq_n_f64(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_f64x2<const SHIFT: usize>(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
if SHIFT >= 2usize {
return b;
}
let result = dyn_vext_128(
self,
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn abs_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>) -> f64x2<Neon> {
vabsq_f64(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn neg_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>) -> f64x2<Neon> {
vnegq_f64(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn sqrt_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>) -> f64x2<Neon> {
vsqrtq_f64(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn approximate_recip_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>) -> f64x2<Neon> {
vrecpeq_f64(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn add_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>, b: f64x2<Neon>) -> f64x2<Neon> {
vaddq_f64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>, b: f64x2<Neon>) -> f64x2<Neon> {
vsubq_f64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>, b: f64x2<Neon>) -> f64x2<Neon> {
vmulq_f64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn div_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>, b: f64x2<Neon>) -> f64x2<Neon> {
vdivq_f64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn copysign_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>, b: f64x2<Neon>) -> f64x2<Neon> {
let sign_mask = vdupq_n_u64(1 << 63);
vbslq_f64(sign_mask, b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>, b: f64x2<Neon>) -> f64x2<Neon> {
vmaxq_f64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>, b: f64x2<Neon>) -> f64x2<Neon> {
vminq_f64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>, b: f64x2<Neon>) -> f64x2<Neon> {
vmaxnmq_f64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn min_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>, b: f64x2<Neon>) -> f64x2<Neon> {
vminnmq_f64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_eq_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>, b: f64x2<Neon>) -> mask64x2<Neon> {
vreinterpretq_s64_u64(vceqq_f64(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>, b: f64x2<Neon>) -> mask64x2<Neon> {
vreinterpretq_s64_u64(vcltq_f64(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>, b: f64x2<Neon>) -> mask64x2<Neon> {
vreinterpretq_s64_u64(vcleq_f64(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>, b: f64x2<Neon>) -> f64x2<Neon> {
let x = a.into();
let y = b.into();
vzip1q_f64(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>, b: f64x2<Neon>) -> f64x2<Neon> {
let x = a.into();
let y = b.into();
vzip2q_f64(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>, b: f64x2<Neon>) -> f64x2<Neon> {
let x = a.into();
let y = b.into();
vuzp1q_f64(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>, b: f64x2<Neon>) -> f64x2<Neon> {
let x = a.into();
let y = b.into();
vuzp2q_f64(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> (f64x2<Self>, f64x2<Self>) {
(self.zip_low_f64x2(a, b), self.zip_high_f64x2(a, b))
}
#[inline(always)]
fn deinterleave_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> (f64x2<Self>, f64x2<Self>) {
(self.unzip_low_f64x2(a, b), self.unzip_high_f64x2(a, b))
}
#[inline(always)]
fn mul_add_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>, b: f64x2<Neon>, c: f64x2<Neon>) -> f64x2<Neon> {
vfmaq_f64(c.into(), b.into(), a.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn mul_sub_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>, b: f64x2<Neon>, c: f64x2<Neon>) -> f64x2<Neon> {
vnegq_f64(vfmsq_f64(c.into(), b.into(), a.into())).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn floor_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>) -> f64x2<Neon> {
vrndmq_f64(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn ceil_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>) -> f64x2<Neon> {
vrndpq_f64(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn round_ties_even_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>) -> f64x2<Neon> {
vrndnq_f64(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn fract_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>) -> f64x2<Neon> {
let c1 = vcvtq_s64_f64(a.into());
let c2 = vcvtq_f64_s64(c1);
vsubq_f64(a.into(), c2).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn trunc_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>) -> f64x2<Neon> {
vrndq_f64(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn select_f64x2(self, a: mask64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Neon,
a: mask64x2<Neon>,
b: f64x2<Neon>,
c: f64x2<Neon>,
) -> f64x2<Neon> {
vbslq_f64(vreinterpretq_u64_s64(a.into()), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x4<Self> {
f64x4 {
val: crate::support::Aligned256(float64x2x2_t(a.val.0, b.val.0)),
simd: self,
}
}
#[inline(always)]
fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2<Self>; 4usize] {
let native = unsafe { vld4q_f64(src.as_ptr()) };
[
native.0.simd_into(self),
native.1.simd_into(self),
native.2.simd_into(self),
native.3.simd_into(self),
]
}
#[inline(always)]
fn store_four_interleaved_f64x2(
self,
vectors: [f64x2<Self>; 4usize],
dest: &mut [f64; 8usize],
) -> () {
let v0: float64x2_t = vectors[0usize].into();
let v1: float64x2_t = vectors[1usize].into();
let v2: float64x2_t = vectors[2usize].into();
let v3: float64x2_t = vectors[3usize].into();
unsafe {
vst4q_f64(dest.as_mut_ptr(), float64x2x4_t(v0, v1, v2, v3));
}
}
#[inline(always)]
fn narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>, b: f64x2<Neon>) -> f32x4<Neon> {
vcvt_high_f32_f64(vcvt_f32_f64(a.into()), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
self.narrow_f64x2(a, b)
}
#[inline(always)]
fn relaxed_narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
self.narrow_f64x2(a, b)
}
#[inline(always)]
fn cvt_u64_f64x2(self, a: f64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>) -> u64x2<Neon> {
vcvtq_u64_f64(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_u64_precise_f64x2(self, a: f64x2<Self>) -> u64x2<Self> {
self.cvt_u64_f64x2(a)
}
#[inline(always)]
fn cvt_i64_f64x2(self, a: f64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: f64x2<Neon>) -> i64x2<Neon> {
vcvtq_s64_f64(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn cvt_i64_precise_f64x2(self, a: f64x2<Self>) -> i64x2<Self> {
self.cvt_i64_f64x2(a)
}
#[inline(always)]
fn splat_i64x2(self, val: i64) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, val: i64) -> i64x2<Neon> {
vdupq_n_s64(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_i64x2<const SHIFT: usize>(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
if SHIFT >= 2usize {
return b;
}
let result = dyn_vext_128(
self,
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>, b: i64x2<Neon>) -> i64x2<Neon> {
vaddq_s64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>, b: i64x2<Neon>) -> i64x2<Neon> {
vsubq_s64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
[
i64::wrapping_mul(a[0usize], b[0usize]),
i64::wrapping_mul(a[1usize], b[1usize]),
]
.simd_into(self)
}
#[inline(always)]
fn and_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>, b: i64x2<Neon>) -> i64x2<Neon> {
vandq_s64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>, b: i64x2<Neon>) -> i64x2<Neon> {
vorrq_s64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>, b: i64x2<Neon>) -> i64x2<Neon> {
veorq_s64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>) -> i64x2<Neon> {
vreinterpretq_s64_s32(vmvnq_s32(vreinterpretq_s32_s64(a.into()))).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn shl_i64x2(self, a: i64x2<Self>, shift: u32) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>, shift: u32) -> i64x2<Neon> {
vshlq_s64(a.into(), vdupq_n_s64(shift as i64)).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>, b: i64x2<Neon>) -> i64x2<Neon> {
vshlq_s64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_i64x2(self, a: i64x2<Self>, shift: u32) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>, shift: u32) -> i64x2<Neon> {
vshlq_s64(a.into(), vdupq_n_s64(-(shift as i64))).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>, b: i64x2<Neon>) -> i64x2<Neon> {
vshlq_s64(a.into(), vnegq_s64(b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
[
i64::max(a[0usize], b[0usize]),
i64::max(a[1usize], b[1usize]),
]
.simd_into(self)
}
#[inline(always)]
fn min_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
[
i64::min(a[0usize], b[0usize]),
i64::min(a[1usize], b[1usize]),
]
.simd_into(self)
}
#[inline(always)]
fn simd_eq_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>, b: i64x2<Neon>) -> mask64x2<Neon> {
vreinterpretq_s64_u64(vceqq_s64(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>, b: i64x2<Neon>) -> mask64x2<Neon> {
vreinterpretq_s64_u64(vcltq_s64(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>, b: i64x2<Neon>) -> mask64x2<Neon> {
vreinterpretq_s64_u64(vcleq_s64(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>, b: i64x2<Neon>) -> i64x2<Neon> {
let x = a.into();
let y = b.into();
vzip1q_s64(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>, b: i64x2<Neon>) -> i64x2<Neon> {
let x = a.into();
let y = b.into();
vzip2q_s64(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>, b: i64x2<Neon>) -> i64x2<Neon> {
let x = a.into();
let y = b.into();
vuzp1q_s64(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>, b: i64x2<Neon>) -> i64x2<Neon> {
let x = a.into();
let y = b.into();
vuzp2q_s64(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> (i64x2<Self>, i64x2<Self>) {
(self.zip_low_i64x2(a, b), self.zip_high_i64x2(a, b))
}
#[inline(always)]
fn deinterleave_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> (i64x2<Self>, i64x2<Self>) {
(self.unzip_low_i64x2(a, b), self.unzip_high_i64x2(a, b))
}
#[inline(always)]
fn select_i64x2(self, a: mask64x2<Self>, b: i64x2<Self>, c: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Neon,
a: mask64x2<Neon>,
b: i64x2<Neon>,
c: i64x2<Neon>,
) -> i64x2<Neon> {
vbslq_s64(vreinterpretq_u64_s64(a.into()), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x4<Self> {
i64x4 {
val: crate::support::Aligned256(int64x2x2_t(a.val.0, b.val.0)),
simd: self,
}
}
#[inline(always)]
fn neg_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>) -> i64x2<Neon> {
vnegq_s64(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2<Self>; 4usize] {
let native = unsafe { vld4q_s64(src.as_ptr()) };
[
native.0.simd_into(self),
native.1.simd_into(self),
native.2.simd_into(self),
native.3.simd_into(self),
]
}
#[inline(always)]
fn store_four_interleaved_i64x2(
self,
vectors: [i64x2<Self>; 4usize],
dest: &mut [i64; 8usize],
) -> () {
let v0: int64x2_t = vectors[0usize].into();
let v1: int64x2_t = vectors[1usize].into();
let v2: int64x2_t = vectors[2usize].into();
let v3: int64x2_t = vectors[3usize].into();
unsafe {
vst4q_s64(dest.as_mut_ptr(), int64x2x4_t(v0, v1, v2, v3));
}
}
#[inline(always)]
fn narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>, b: i64x2<Neon>) -> i32x4<Neon> {
vcombine_s32(vmovn_s64(a.into()), vmovn_s64(b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>, b: i64x2<Neon>) -> i32x4<Neon> {
vcombine_s32(vqmovn_s64(a.into()), vqmovn_s64(b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn relaxed_narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= i32::MIN as i64 && value <= i32::MAX as i64 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.narrow_i64x2(a, b)
}
#[inline(always)]
fn cvt_f64_i64x2(self, a: i64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: i64x2<Neon>) -> f64x2<Neon> {
vcvtq_f64_s64(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_u64x2(self, val: u64) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, val: u64) -> u64x2<Neon> {
vdupq_n_u64(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn slide_u64x2<const SHIFT: usize>(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
if SHIFT >= 2usize {
return b;
}
let result = dyn_vext_128(
self,
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u64x2<Neon>, b: u64x2<Neon>) -> u64x2<Neon> {
vaddq_u64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn sub_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u64x2<Neon>, b: u64x2<Neon>) -> u64x2<Neon> {
vsubq_u64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn mul_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
[
u64::wrapping_mul(a[0usize], b[0usize]),
u64::wrapping_mul(a[1usize], b[1usize]),
]
.simd_into(self)
}
#[inline(always)]
fn and_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u64x2<Neon>, b: u64x2<Neon>) -> u64x2<Neon> {
vandq_u64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u64x2<Neon>, b: u64x2<Neon>) -> u64x2<Neon> {
vorrq_u64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u64x2<Neon>, b: u64x2<Neon>) -> u64x2<Neon> {
veorq_u64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u64x2<Neon>) -> u64x2<Neon> {
vreinterpretq_u64_u32(vmvnq_u32(vreinterpretq_u32_u64(a.into()))).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn shl_u64x2(self, a: u64x2<Self>, shift: u32) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u64x2<Neon>, shift: u32) -> u64x2<Neon> {
vshlq_u64(a.into(), vdupq_n_s64(shift as i64)).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shlv_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u64x2<Neon>, b: u64x2<Neon>) -> u64x2<Neon> {
vshlq_u64(a.into(), vreinterpretq_s64_u64(b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn shr_u64x2(self, a: u64x2<Self>, shift: u32) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u64x2<Neon>, shift: u32) -> u64x2<Neon> {
vshlq_u64(a.into(), vdupq_n_s64(-(shift as i64))).simd_into(token)
}
);
kernel(self, a, shift)
}
#[inline(always)]
fn shrv_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u64x2<Neon>, b: u64x2<Neon>) -> u64x2<Neon> {
vshlq_u64(a.into(), vnegq_s64(vreinterpretq_s64_u64(b.into()))).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn max_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
[
u64::max(a[0usize], b[0usize]),
u64::max(a[1usize], b[1usize]),
]
.simd_into(self)
}
#[inline(always)]
fn min_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
[
u64::min(a[0usize], b[0usize]),
u64::min(a[1usize], b[1usize]),
]
.simd_into(self)
}
#[inline(always)]
fn simd_eq_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u64x2<Neon>, b: u64x2<Neon>) -> mask64x2<Neon> {
vreinterpretq_s64_u64(vceqq_u64(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_lt_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u64x2<Neon>, b: u64x2<Neon>) -> mask64x2<Neon> {
vreinterpretq_s64_u64(vcltq_u64(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn simd_le_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u64x2<Neon>, b: u64x2<Neon>) -> mask64x2<Neon> {
vreinterpretq_s64_u64(vcleq_u64(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_low_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u64x2<Neon>, b: u64x2<Neon>) -> u64x2<Neon> {
let x = a.into();
let y = b.into();
vzip1q_u64(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn zip_high_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u64x2<Neon>, b: u64x2<Neon>) -> u64x2<Neon> {
let x = a.into();
let y = b.into();
vzip2q_u64(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_low_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u64x2<Neon>, b: u64x2<Neon>) -> u64x2<Neon> {
let x = a.into();
let y = b.into();
vuzp1q_u64(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn unzip_high_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u64x2<Neon>, b: u64x2<Neon>) -> u64x2<Neon> {
let x = a.into();
let y = b.into();
vuzp2q_u64(x, y).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn interleave_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> (u64x2<Self>, u64x2<Self>) {
(self.zip_low_u64x2(a, b), self.zip_high_u64x2(a, b))
}
#[inline(always)]
fn deinterleave_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> (u64x2<Self>, u64x2<Self>) {
(self.unzip_low_u64x2(a, b), self.unzip_high_u64x2(a, b))
}
#[inline(always)]
fn select_u64x2(self, a: mask64x2<Self>, b: u64x2<Self>, c: u64x2<Self>) -> u64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Neon,
a: mask64x2<Neon>,
b: u64x2<Neon>,
c: u64x2<Neon>,
) -> u64x2<Neon> {
vbslq_u64(vreinterpretq_u64_s64(a.into()), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn combine_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x4<Self> {
u64x4 {
val: crate::support::Aligned256(uint64x2x2_t(a.val.0, b.val.0)),
simd: self,
}
}
#[inline(always)]
fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2<Self>; 4usize] {
let native = unsafe { vld4q_u64(src.as_ptr()) };
[
native.0.simd_into(self),
native.1.simd_into(self),
native.2.simd_into(self),
native.3.simd_into(self),
]
}
#[inline(always)]
fn store_four_interleaved_u64x2(
self,
vectors: [u64x2<Self>; 4usize],
dest: &mut [u64; 8usize],
) -> () {
let v0: uint64x2_t = vectors[0usize].into();
let v1: uint64x2_t = vectors[1usize].into();
let v2: uint64x2_t = vectors[2usize].into();
let v3: uint64x2_t = vectors[3usize].into();
unsafe {
vst4q_u64(dest.as_mut_ptr(), uint64x2x4_t(v0, v1, v2, v3));
}
}
#[inline(always)]
fn narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u64x2<Neon>, b: u64x2<Neon>) -> u32x4<Neon> {
vcombine_u32(vmovn_u64(a.into()), vmovn_u64(b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn saturating_narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u64x2<Neon>, b: u64x2<Neon>) -> u32x4<Neon> {
vcombine_u32(vqmovn_u64(a.into()), vqmovn_u64(b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn relaxed_narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= u32::MIN as u64 && value <= u32::MAX as u64 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.narrow_u64x2(a, b)
}
#[inline(always)]
fn cvt_f64_u64x2(self, a: u64x2<Self>) -> f64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u64x2<Neon>) -> f64x2<Neon> {
vcvtq_f64_u64(a.into()).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn splat_mask64x2(self, val: bool) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, val: bool) -> mask64x2<Neon> {
let val: i64 = if val { !0 } else { 0 };
vdupq_n_s64(val).simd_into(token)
}
);
kernel(self, val)
}
#[inline(always)]
fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, bits: u64) -> mask64x2<Neon> {
let shifts =
crate::transmute::checked_transmute_copy::<[i64; 2], int64x2_t>(&[63, 62]);
let shifted = vshlq_u64(vdupq_n_u64(bits), shifts);
let mask = vcltq_s64(vreinterpretq_s64_u64(shifted), vdupq_n_s64(0));
vreinterpretq_s64_u64(mask).simd_into(token)
}
);
kernel(self, bits)
}
#[inline(always)]
fn to_bitmask_mask64x2(self, a: mask64x2<Self>) -> u64 {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask64x2<Neon>) -> u64 {
let weights =
crate::transmute::checked_transmute_copy::<[u64; 2], uint64x2_t>(&[1, 2]);
let bits = vandq_u64(vreinterpretq_u64_s64(a.into()), weights);
vaddvq_u64(bits)
}
);
kernel(self, a)
}
#[inline(always)]
fn set_mask64x2(self, a: &mut mask64x2<Self>, index: usize, value: bool) -> () {
assert!(
index < 2usize,
"mask lane index {index} is out of bounds for {} lanes",
2usize
);
let mut lanes: [i64; 2usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn and_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask64x2<Neon>, b: mask64x2<Neon>) -> mask64x2<Neon> {
vandq_s64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn or_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask64x2<Neon>, b: mask64x2<Neon>) -> mask64x2<Neon> {
vorrq_s64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn xor_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask64x2<Neon>, b: mask64x2<Neon>) -> mask64x2<Neon> {
veorq_s64(a.into(), b.into()).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn not_mask64x2(self, a: mask64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask64x2<Neon>) -> mask64x2<Neon> {
vreinterpretq_s64_s32(vmvnq_s32(vreinterpretq_s32_s64(a.into()))).simd_into(token)
}
);
kernel(self, a)
}
#[inline(always)]
fn select_mask64x2(
self,
a: mask64x2<Self>,
b: mask64x2<Self>,
c: mask64x2<Self>,
) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(
token: Neon,
a: mask64x2<Neon>,
b: mask64x2<Neon>,
c: mask64x2<Neon>,
) -> mask64x2<Neon> {
vbslq_s64(vreinterpretq_u64_s64(a.into()), b.into(), c.into()).simd_into(token)
}
);
kernel(self, a, b, c)
}
#[inline(always)]
fn simd_eq_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask64x2<Neon>, b: mask64x2<Neon>) -> mask64x2<Neon> {
vreinterpretq_s64_u64(vceqq_s64(a.into(), b.into())).simd_into(token)
}
);
kernel(self, a, b)
}
#[inline(always)]
fn any_true_mask64x2(self, a: mask64x2<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask64x2<Neon>) -> bool {
vmaxvq_u32(vreinterpretq_u32_s64(a.into())) != 0
}
);
kernel(self, a)
}
#[inline(always)]
fn all_true_mask64x2(self, a: mask64x2<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask64x2<Neon>) -> bool {
vminvq_u32(vreinterpretq_u32_s64(a.into())) == 0xffffffff
}
);
kernel(self, a)
}
#[inline(always)]
fn any_false_mask64x2(self, a: mask64x2<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask64x2<Neon>) -> bool {
vminvq_u32(vreinterpretq_u32_s64(a.into())) != 0xffffffff
}
);
kernel(self, a)
}
#[inline(always)]
fn all_false_mask64x2(self, a: mask64x2<Self>) -> bool {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: mask64x2<Neon>) -> bool {
vmaxvq_u32(vreinterpretq_u32_s64(a.into())) == 0
}
);
kernel(self, a)
}
#[inline(always)]
fn combine_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x4<Self> {
mask64x4 {
val: crate::support::Aligned256(int64x2x2_t(a.val.0, b.val.0)),
simd: self,
}
}
#[inline(always)]
fn slide_f32x8<const SHIFT: usize>(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = {
let a_bytes = Bytes::to_bytes(a).val.0;
let b_bytes = Bytes::to_bytes(b).val.0;
let a_blocks = [a_bytes.0, a_bytes.1];
let b_blocks = [b_bytes.0, b_bytes.1];
let shift_bytes = SHIFT * 4usize;
uint8x16x2_t(
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
0,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
1,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
)
};
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn combine_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x16<Self> {
f32x16 {
val: crate::support::Aligned512(float32x4x4_t(
a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1,
)),
simd: self,
}
}
#[inline(always)]
fn split_f32x8(self, a: f32x8<Self>) -> (f32x4<Self>, f32x4<Self>) {
(
f32x4 {
val: crate::support::Aligned128(a.val.0.0),
simd: self,
},
f32x4 {
val: crate::support::Aligned128(a.val.0.1),
simd: self,
},
)
}
#[inline(always)]
fn slide_i8x32<const SHIFT: usize>(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
if SHIFT >= 32usize {
return b;
}
let result = {
let a_bytes = Bytes::to_bytes(a).val.0;
let b_bytes = Bytes::to_bytes(b).val.0;
let a_blocks = [a_bytes.0, a_bytes.1];
let b_blocks = [b_bytes.0, b_bytes.1];
let shift_bytes = SHIFT;
uint8x16x2_t(
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
0,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
1,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
)
};
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn combine_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x64<Self> {
i8x64 {
val: crate::support::Aligned512(int8x16x4_t(
a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1,
)),
simd: self,
}
}
#[inline(always)]
fn split_i8x32(self, a: i8x32<Self>) -> (i8x16<Self>, i8x16<Self>) {
(
i8x16 {
val: crate::support::Aligned128(a.val.0.0),
simd: self,
},
i8x16 {
val: crate::support::Aligned128(a.val.0.1),
simd: self,
},
)
}
#[inline(always)]
fn slide_u8x32<const SHIFT: usize>(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
if SHIFT >= 32usize {
return b;
}
let result = {
let a_bytes = Bytes::to_bytes(a).val.0;
let b_bytes = Bytes::to_bytes(b).val.0;
let a_blocks = [a_bytes.0, a_bytes.1];
let b_blocks = [b_bytes.0, b_bytes.1];
let shift_bytes = SHIFT;
uint8x16x2_t(
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
0,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
1,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
)
};
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
self.swizzle_dyn_precise_u8x32(a, indices)
}
#[inline(always)]
fn swizzle_dyn_precise_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x32<Neon>, indices: u8x32<Neon>) -> u8x32<Neon> {
let table = Bytes::to_bytes(a).val.0;
let indices: uint8x16x2_t = indices.into();
let result =
uint8x16x2_t(vqtbl2q_u8(table, indices.0), vqtbl2q_u8(table, indices.1));
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn combine_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x64<Self> {
u8x64 {
val: crate::support::Aligned512(uint8x16x4_t(
a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1,
)),
simd: self,
}
}
#[inline(always)]
fn split_u8x32(self, a: u8x32<Self>) -> (u8x16<Self>, u8x16<Self>) {
(
u8x16 {
val: crate::support::Aligned128(a.val.0.0),
simd: self,
},
u8x16 {
val: crate::support::Aligned128(a.val.0.1),
simd: self,
},
)
}
#[inline(always)]
fn set_mask8x32(self, a: &mut mask8x32<Self>, index: usize, value: bool) -> () {
assert!(
index < 32usize,
"mask lane index {index} is out of bounds for {} lanes",
32usize
);
let mut lanes: [i8; 32usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn combine_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x64<Self> {
mask8x64 {
val: crate::support::Aligned512(int8x16x4_t(
a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1,
)),
simd: self,
}
}
#[inline(always)]
fn split_mask8x32(self, a: mask8x32<Self>) -> (mask8x16<Self>, mask8x16<Self>) {
(
mask8x16 {
val: crate::support::Aligned128(a.val.0.0),
simd: self,
},
mask8x16 {
val: crate::support::Aligned128(a.val.0.1),
simd: self,
},
)
}
#[inline(always)]
fn slide_i16x16<const SHIFT: usize>(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = {
let a_bytes = Bytes::to_bytes(a).val.0;
let b_bytes = Bytes::to_bytes(b).val.0;
let a_blocks = [a_bytes.0, a_bytes.1];
let b_blocks = [b_bytes.0, b_bytes.1];
let shift_bytes = SHIFT * 2usize;
uint8x16x2_t(
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
0,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
1,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
)
};
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn combine_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x32<Self> {
i16x32 {
val: crate::support::Aligned512(int16x8x4_t(
a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1,
)),
simd: self,
}
}
#[inline(always)]
fn split_i16x16(self, a: i16x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
(
i16x8 {
val: crate::support::Aligned128(a.val.0.0),
simd: self,
},
i16x8 {
val: crate::support::Aligned128(a.val.0.1),
simd: self,
},
)
}
#[inline(always)]
fn slide_u16x16<const SHIFT: usize>(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = {
let a_bytes = Bytes::to_bytes(a).val.0;
let b_bytes = Bytes::to_bytes(b).val.0;
let a_blocks = [a_bytes.0, a_bytes.1];
let b_blocks = [b_bytes.0, b_bytes.1];
let shift_bytes = SHIFT * 2usize;
uint8x16x2_t(
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
0,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
1,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
)
};
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn combine_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x32<Self> {
u16x32 {
val: crate::support::Aligned512(uint16x8x4_t(
a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1,
)),
simd: self,
}
}
#[inline(always)]
fn split_u16x16(self, a: u16x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
(
u16x8 {
val: crate::support::Aligned128(a.val.0.0),
simd: self,
},
u16x8 {
val: crate::support::Aligned128(a.val.0.1),
simd: self,
},
)
}
#[inline(always)]
fn set_mask16x16(self, a: &mut mask16x16<Self>, index: usize, value: bool) -> () {
assert!(
index < 16usize,
"mask lane index {index} is out of bounds for {} lanes",
16usize
);
let mut lanes: [i16; 16usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn combine_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x32<Self> {
mask16x32 {
val: crate::support::Aligned512(int16x8x4_t(
a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1,
)),
simd: self,
}
}
#[inline(always)]
fn split_mask16x16(self, a: mask16x16<Self>) -> (mask16x8<Self>, mask16x8<Self>) {
(
mask16x8 {
val: crate::support::Aligned128(a.val.0.0),
simd: self,
},
mask16x8 {
val: crate::support::Aligned128(a.val.0.1),
simd: self,
},
)
}
#[inline(always)]
fn slide_i32x8<const SHIFT: usize>(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = {
let a_bytes = Bytes::to_bytes(a).val.0;
let b_bytes = Bytes::to_bytes(b).val.0;
let a_blocks = [a_bytes.0, a_bytes.1];
let b_blocks = [b_bytes.0, b_bytes.1];
let shift_bytes = SHIFT * 4usize;
uint8x16x2_t(
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
0,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
1,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
)
};
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn combine_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x16<Self> {
i32x16 {
val: crate::support::Aligned512(int32x4x4_t(
a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1,
)),
simd: self,
}
}
#[inline(always)]
fn split_i32x8(self, a: i32x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
(
i32x4 {
val: crate::support::Aligned128(a.val.0.0),
simd: self,
},
i32x4 {
val: crate::support::Aligned128(a.val.0.1),
simd: self,
},
)
}
#[inline(always)]
fn slide_u32x8<const SHIFT: usize>(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = {
let a_bytes = Bytes::to_bytes(a).val.0;
let b_bytes = Bytes::to_bytes(b).val.0;
let a_blocks = [a_bytes.0, a_bytes.1];
let b_blocks = [b_bytes.0, b_bytes.1];
let shift_bytes = SHIFT * 4usize;
uint8x16x2_t(
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
0,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
1,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
)
};
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn combine_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x16<Self> {
u32x16 {
val: crate::support::Aligned512(uint32x4x4_t(
a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1,
)),
simd: self,
}
}
#[inline(always)]
fn split_u32x8(self, a: u32x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
(
u32x4 {
val: crate::support::Aligned128(a.val.0.0),
simd: self,
},
u32x4 {
val: crate::support::Aligned128(a.val.0.1),
simd: self,
},
)
}
#[inline(always)]
fn set_mask32x8(self, a: &mut mask32x8<Self>, index: usize, value: bool) -> () {
assert!(
index < 8usize,
"mask lane index {index} is out of bounds for {} lanes",
8usize
);
let mut lanes: [i32; 8usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn combine_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x16<Self> {
mask32x16 {
val: crate::support::Aligned512(int32x4x4_t(
a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1,
)),
simd: self,
}
}
#[inline(always)]
fn split_mask32x8(self, a: mask32x8<Self>) -> (mask32x4<Self>, mask32x4<Self>) {
(
mask32x4 {
val: crate::support::Aligned128(a.val.0.0),
simd: self,
},
mask32x4 {
val: crate::support::Aligned128(a.val.0.1),
simd: self,
},
)
}
#[inline(always)]
fn slide_f64x4<const SHIFT: usize>(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = {
let a_bytes = Bytes::to_bytes(a).val.0;
let b_bytes = Bytes::to_bytes(b).val.0;
let a_blocks = [a_bytes.0, a_bytes.1];
let b_blocks = [b_bytes.0, b_bytes.1];
let shift_bytes = SHIFT * 8usize;
uint8x16x2_t(
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
0,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
1,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
)
};
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn combine_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x8<Self> {
f64x8 {
val: crate::support::Aligned512(float64x2x4_t(
a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1,
)),
simd: self,
}
}
#[inline(always)]
fn split_f64x4(self, a: f64x4<Self>) -> (f64x2<Self>, f64x2<Self>) {
(
f64x2 {
val: crate::support::Aligned128(a.val.0.0),
simd: self,
},
f64x2 {
val: crate::support::Aligned128(a.val.0.1),
simd: self,
},
)
}
#[inline(always)]
fn slide_i64x4<const SHIFT: usize>(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = {
let a_bytes = Bytes::to_bytes(a).val.0;
let b_bytes = Bytes::to_bytes(b).val.0;
let a_blocks = [a_bytes.0, a_bytes.1];
let b_blocks = [b_bytes.0, b_bytes.1];
let shift_bytes = SHIFT * 8usize;
uint8x16x2_t(
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
0,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
1,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
)
};
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn combine_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x8<Self> {
i64x8 {
val: crate::support::Aligned512(int64x2x4_t(
a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1,
)),
simd: self,
}
}
#[inline(always)]
fn split_i64x4(self, a: i64x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
(
i64x2 {
val: crate::support::Aligned128(a.val.0.0),
simd: self,
},
i64x2 {
val: crate::support::Aligned128(a.val.0.1),
simd: self,
},
)
}
#[inline(always)]
fn slide_u64x4<const SHIFT: usize>(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = {
let a_bytes = Bytes::to_bytes(a).val.0;
let b_bytes = Bytes::to_bytes(b).val.0;
let a_blocks = [a_bytes.0, a_bytes.1];
let b_blocks = [b_bytes.0, b_bytes.1];
let shift_bytes = SHIFT * 8usize;
uint8x16x2_t(
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
0,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
1,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
)
};
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn combine_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x8<Self> {
u64x8 {
val: crate::support::Aligned512(uint64x2x4_t(
a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1,
)),
simd: self,
}
}
#[inline(always)]
fn split_u64x4(self, a: u64x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
(
u64x2 {
val: crate::support::Aligned128(a.val.0.0),
simd: self,
},
u64x2 {
val: crate::support::Aligned128(a.val.0.1),
simd: self,
},
)
}
#[inline(always)]
fn set_mask64x4(self, a: &mut mask64x4<Self>, index: usize, value: bool) -> () {
assert!(
index < 4usize,
"mask lane index {index} is out of bounds for {} lanes",
4usize
);
let mut lanes: [i64; 4usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn combine_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x8<Self> {
mask64x8 {
val: crate::support::Aligned512(int64x2x4_t(
a.val.0.0, a.val.0.1, b.val.0.0, b.val.0.1,
)),
simd: self,
}
}
#[inline(always)]
fn split_mask64x4(self, a: mask64x4<Self>) -> (mask64x2<Self>, mask64x2<Self>) {
(
mask64x2 {
val: crate::support::Aligned128(a.val.0.0),
simd: self,
},
mask64x2 {
val: crate::support::Aligned128(a.val.0.1),
simd: self,
},
)
}
#[inline(always)]
fn slide_f32x16<const SHIFT: usize>(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = {
let a_bytes = Bytes::to_bytes(a).val.0;
let b_bytes = Bytes::to_bytes(b).val.0;
let a_blocks = [a_bytes.0, a_bytes.1, a_bytes.2, a_bytes.3];
let b_blocks = [b_bytes.0, b_bytes.1, b_bytes.2, b_bytes.3];
let shift_bytes = SHIFT * 4usize;
uint8x16x4_t(
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
0,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
1,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
2,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
3,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
)
};
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_f32x16(self, a: f32x16<Self>) -> (f32x8<Self>, f32x8<Self>) {
(
f32x8 {
val: crate::support::Aligned256(float32x4x2_t(a.val.0.0, a.val.0.1)),
simd: self,
},
f32x8 {
val: crate::support::Aligned256(float32x4x2_t(a.val.0.2, a.val.0.3)),
simd: self,
},
)
}
#[inline(always)]
fn slide_i8x64<const SHIFT: usize>(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
if SHIFT >= 64usize {
return b;
}
let result = {
let a_bytes = Bytes::to_bytes(a).val.0;
let b_bytes = Bytes::to_bytes(b).val.0;
let a_blocks = [a_bytes.0, a_bytes.1, a_bytes.2, a_bytes.3];
let b_blocks = [b_bytes.0, b_bytes.1, b_bytes.2, b_bytes.3];
let shift_bytes = SHIFT;
uint8x16x4_t(
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
0,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
1,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
2,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
3,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
)
};
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_i8x64(self, a: i8x64<Self>) -> (i8x32<Self>, i8x32<Self>) {
(
i8x32 {
val: crate::support::Aligned256(int8x16x2_t(a.val.0.0, a.val.0.1)),
simd: self,
},
i8x32 {
val: crate::support::Aligned256(int8x16x2_t(a.val.0.2, a.val.0.3)),
simd: self,
},
)
}
#[inline(always)]
fn slide_u8x64<const SHIFT: usize>(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
if SHIFT >= 64usize {
return b;
}
let result = {
let a_bytes = Bytes::to_bytes(a).val.0;
let b_bytes = Bytes::to_bytes(b).val.0;
let a_blocks = [a_bytes.0, a_bytes.1, a_bytes.2, a_bytes.3];
let b_blocks = [b_bytes.0, b_bytes.1, b_bytes.2, b_bytes.3];
let shift_bytes = SHIFT;
uint8x16x4_t(
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
0,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
1,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
2,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
3,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
)
};
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_u8x64(self, a: u8x64<Self>, indices: u8x64<Self>) -> u8x64<Self> {
self.swizzle_dyn_precise_u8x64(a, indices)
}
#[inline(always)]
fn swizzle_dyn_precise_u8x64(self, a: u8x64<Self>, indices: u8x64<Self>) -> u8x64<Self> {
crate::kernel!(
#[inline(always)]
fn kernel(token: Neon, a: u8x64<Neon>, indices: u8x64<Neon>) -> u8x64<Neon> {
let table = Bytes::to_bytes(a).val.0;
let indices: uint8x16x4_t = indices.into();
let result = uint8x16x4_t(
vqtbl4q_u8(table, indices.0),
vqtbl4q_u8(table, indices.1),
vqtbl4q_u8(table, indices.2),
vqtbl4q_u8(table, indices.3),
);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: token,
})
}
);
kernel(self, a, indices)
}
#[inline(always)]
fn split_u8x64(self, a: u8x64<Self>) -> (u8x32<Self>, u8x32<Self>) {
(
u8x32 {
val: crate::support::Aligned256(uint8x16x2_t(a.val.0.0, a.val.0.1)),
simd: self,
},
u8x32 {
val: crate::support::Aligned256(uint8x16x2_t(a.val.0.2, a.val.0.3)),
simd: self,
},
)
}
#[inline(always)]
fn set_mask8x64(self, a: &mut mask8x64<Self>, index: usize, value: bool) -> () {
assert!(
index < 64usize,
"mask lane index {index} is out of bounds for {} lanes",
64usize
);
let mut lanes: [i8; 64usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn split_mask8x64(self, a: mask8x64<Self>) -> (mask8x32<Self>, mask8x32<Self>) {
(
mask8x32 {
val: crate::support::Aligned256(int8x16x2_t(a.val.0.0, a.val.0.1)),
simd: self,
},
mask8x32 {
val: crate::support::Aligned256(int8x16x2_t(a.val.0.2, a.val.0.3)),
simd: self,
},
)
}
#[inline(always)]
fn slide_i16x32<const SHIFT: usize>(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
if SHIFT >= 32usize {
return b;
}
let result = {
let a_bytes = Bytes::to_bytes(a).val.0;
let b_bytes = Bytes::to_bytes(b).val.0;
let a_blocks = [a_bytes.0, a_bytes.1, a_bytes.2, a_bytes.3];
let b_blocks = [b_bytes.0, b_bytes.1, b_bytes.2, b_bytes.3];
let shift_bytes = SHIFT * 2usize;
uint8x16x4_t(
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
0,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
1,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
2,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
3,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
)
};
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_i16x32(self, a: i16x32<Self>) -> (i16x16<Self>, i16x16<Self>) {
(
i16x16 {
val: crate::support::Aligned256(int16x8x2_t(a.val.0.0, a.val.0.1)),
simd: self,
},
i16x16 {
val: crate::support::Aligned256(int16x8x2_t(a.val.0.2, a.val.0.3)),
simd: self,
},
)
}
#[inline(always)]
fn slide_u16x32<const SHIFT: usize>(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
if SHIFT >= 32usize {
return b;
}
let result = {
let a_bytes = Bytes::to_bytes(a).val.0;
let b_bytes = Bytes::to_bytes(b).val.0;
let a_blocks = [a_bytes.0, a_bytes.1, a_bytes.2, a_bytes.3];
let b_blocks = [b_bytes.0, b_bytes.1, b_bytes.2, b_bytes.3];
let shift_bytes = SHIFT * 2usize;
uint8x16x4_t(
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
0,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
1,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
2,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
3,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
)
};
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_u16x32(self, a: u16x32<Self>) -> (u16x16<Self>, u16x16<Self>) {
(
u16x16 {
val: crate::support::Aligned256(uint16x8x2_t(a.val.0.0, a.val.0.1)),
simd: self,
},
u16x16 {
val: crate::support::Aligned256(uint16x8x2_t(a.val.0.2, a.val.0.3)),
simd: self,
},
)
}
#[inline(always)]
fn set_mask16x32(self, a: &mut mask16x32<Self>, index: usize, value: bool) -> () {
assert!(
index < 32usize,
"mask lane index {index} is out of bounds for {} lanes",
32usize
);
let mut lanes: [i16; 32usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn split_mask16x32(self, a: mask16x32<Self>) -> (mask16x16<Self>, mask16x16<Self>) {
(
mask16x16 {
val: crate::support::Aligned256(int16x8x2_t(a.val.0.0, a.val.0.1)),
simd: self,
},
mask16x16 {
val: crate::support::Aligned256(int16x8x2_t(a.val.0.2, a.val.0.3)),
simd: self,
},
)
}
#[inline(always)]
fn slide_i32x16<const SHIFT: usize>(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = {
let a_bytes = Bytes::to_bytes(a).val.0;
let b_bytes = Bytes::to_bytes(b).val.0;
let a_blocks = [a_bytes.0, a_bytes.1, a_bytes.2, a_bytes.3];
let b_blocks = [b_bytes.0, b_bytes.1, b_bytes.2, b_bytes.3];
let shift_bytes = SHIFT * 4usize;
uint8x16x4_t(
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
0,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
1,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
2,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
3,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
)
};
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_i32x16(self, a: i32x16<Self>) -> (i32x8<Self>, i32x8<Self>) {
(
i32x8 {
val: crate::support::Aligned256(int32x4x2_t(a.val.0.0, a.val.0.1)),
simd: self,
},
i32x8 {
val: crate::support::Aligned256(int32x4x2_t(a.val.0.2, a.val.0.3)),
simd: self,
},
)
}
#[inline(always)]
fn slide_u32x16<const SHIFT: usize>(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = {
let a_bytes = Bytes::to_bytes(a).val.0;
let b_bytes = Bytes::to_bytes(b).val.0;
let a_blocks = [a_bytes.0, a_bytes.1, a_bytes.2, a_bytes.3];
let b_blocks = [b_bytes.0, b_bytes.1, b_bytes.2, b_bytes.3];
let shift_bytes = SHIFT * 4usize;
uint8x16x4_t(
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
0,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
1,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
2,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
3,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
)
};
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_u32x16(self, a: u32x16<Self>) -> (u32x8<Self>, u32x8<Self>) {
(
u32x8 {
val: crate::support::Aligned256(uint32x4x2_t(a.val.0.0, a.val.0.1)),
simd: self,
},
u32x8 {
val: crate::support::Aligned256(uint32x4x2_t(a.val.0.2, a.val.0.3)),
simd: self,
},
)
}
#[inline(always)]
fn set_mask32x16(self, a: &mut mask32x16<Self>, index: usize, value: bool) -> () {
assert!(
index < 16usize,
"mask lane index {index} is out of bounds for {} lanes",
16usize
);
let mut lanes: [i32; 16usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn split_mask32x16(self, a: mask32x16<Self>) -> (mask32x8<Self>, mask32x8<Self>) {
(
mask32x8 {
val: crate::support::Aligned256(int32x4x2_t(a.val.0.0, a.val.0.1)),
simd: self,
},
mask32x8 {
val: crate::support::Aligned256(int32x4x2_t(a.val.0.2, a.val.0.3)),
simd: self,
},
)
}
#[inline(always)]
fn slide_f64x8<const SHIFT: usize>(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = {
let a_bytes = Bytes::to_bytes(a).val.0;
let b_bytes = Bytes::to_bytes(b).val.0;
let a_blocks = [a_bytes.0, a_bytes.1, a_bytes.2, a_bytes.3];
let b_blocks = [b_bytes.0, b_bytes.1, b_bytes.2, b_bytes.3];
let shift_bytes = SHIFT * 8usize;
uint8x16x4_t(
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
0,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
1,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
2,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
3,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
)
};
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_f64x8(self, a: f64x8<Self>) -> (f64x4<Self>, f64x4<Self>) {
(
f64x4 {
val: crate::support::Aligned256(float64x2x2_t(a.val.0.0, a.val.0.1)),
simd: self,
},
f64x4 {
val: crate::support::Aligned256(float64x2x2_t(a.val.0.2, a.val.0.3)),
simd: self,
},
)
}
#[inline(always)]
fn slide_i64x8<const SHIFT: usize>(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = {
let a_bytes = Bytes::to_bytes(a).val.0;
let b_bytes = Bytes::to_bytes(b).val.0;
let a_blocks = [a_bytes.0, a_bytes.1, a_bytes.2, a_bytes.3];
let b_blocks = [b_bytes.0, b_bytes.1, b_bytes.2, b_bytes.3];
let shift_bytes = SHIFT * 8usize;
uint8x16x4_t(
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
0,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
1,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
2,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
3,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
)
};
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_i64x8(self, a: i64x8<Self>) -> (i64x4<Self>, i64x4<Self>) {
(
i64x4 {
val: crate::support::Aligned256(int64x2x2_t(a.val.0.0, a.val.0.1)),
simd: self,
},
i64x4 {
val: crate::support::Aligned256(int64x2x2_t(a.val.0.2, a.val.0.3)),
simd: self,
},
)
}
#[inline(always)]
fn slide_u64x8<const SHIFT: usize>(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = {
let a_bytes = Bytes::to_bytes(a).val.0;
let b_bytes = Bytes::to_bytes(b).val.0;
let a_blocks = [a_bytes.0, a_bytes.1, a_bytes.2, a_bytes.3];
let b_blocks = [b_bytes.0, b_bytes.1, b_bytes.2, b_bytes.3];
let shift_bytes = SHIFT * 8usize;
uint8x16x4_t(
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
0,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
1,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
2,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(
&a_blocks,
&b_blocks,
3,
shift_bytes,
);
dyn_vext_128(self, lo, hi, shift_bytes % 16)
},
)
};
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_u64x8(self, a: u64x8<Self>) -> (u64x4<Self>, u64x4<Self>) {
(
u64x4 {
val: crate::support::Aligned256(uint64x2x2_t(a.val.0.0, a.val.0.1)),
simd: self,
},
u64x4 {
val: crate::support::Aligned256(uint64x2x2_t(a.val.0.2, a.val.0.3)),
simd: self,
},
)
}
#[inline(always)]
fn set_mask64x8(self, a: &mut mask64x8<Self>, index: usize, value: bool) -> () {
assert!(
index < 8usize,
"mask lane index {index} is out of bounds for {} lanes",
8usize
);
let mut lanes: [i64; 8usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn split_mask64x8(self, a: mask64x8<Self>) -> (mask64x4<Self>, mask64x4<Self>) {
(
mask64x4 {
val: crate::support::Aligned256(int64x2x2_t(a.val.0.0, a.val.0.1)),
simd: self,
},
mask64x4 {
val: crate::support::Aligned256(int64x2x2_t(a.val.0.2, a.val.0.3)),
simd: self,
},
)
}
}
impl<S: Simd> SimdFrom<float32x4_t, S> for f32x4<S> {
#[inline(always)]
fn simd_from(simd: S, arch: float32x4_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<f32x4<S>> for float32x4_t {
#[inline(always)]
fn from(value: f32x4<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int8x16_t, S> for i8x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int8x16_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i8x16<S>> for int8x16_t {
#[inline(always)]
fn from(value: i8x16<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<uint8x16_t, S> for u8x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: uint8x16_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u8x16<S>> for uint8x16_t {
#[inline(always)]
fn from(value: u8x16<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int8x16_t, S> for mask8x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int8x16_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<mask8x16<S>> for int8x16_t {
#[inline(always)]
fn from(value: mask8x16<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int16x8_t, S> for i16x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int16x8_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i16x8<S>> for int16x8_t {
#[inline(always)]
fn from(value: i16x8<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<uint16x8_t, S> for u16x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: uint16x8_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u16x8<S>> for uint16x8_t {
#[inline(always)]
fn from(value: u16x8<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int16x8_t, S> for mask16x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int16x8_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<mask16x8<S>> for int16x8_t {
#[inline(always)]
fn from(value: mask16x8<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int32x4_t, S> for i32x4<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int32x4_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i32x4<S>> for int32x4_t {
#[inline(always)]
fn from(value: i32x4<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<uint32x4_t, S> for u32x4<S> {
#[inline(always)]
fn simd_from(simd: S, arch: uint32x4_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u32x4<S>> for uint32x4_t {
#[inline(always)]
fn from(value: u32x4<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int32x4_t, S> for mask32x4<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int32x4_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<mask32x4<S>> for int32x4_t {
#[inline(always)]
fn from(value: mask32x4<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<float64x2_t, S> for f64x2<S> {
#[inline(always)]
fn simd_from(simd: S, arch: float64x2_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<f64x2<S>> for float64x2_t {
#[inline(always)]
fn from(value: f64x2<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int64x2_t, S> for i64x2<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int64x2_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i64x2<S>> for int64x2_t {
#[inline(always)]
fn from(value: i64x2<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<uint64x2_t, S> for u64x2<S> {
#[inline(always)]
fn simd_from(simd: S, arch: uint64x2_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u64x2<S>> for uint64x2_t {
#[inline(always)]
fn from(value: u64x2<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int64x2_t, S> for mask64x2<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int64x2_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<mask64x2<S>> for int64x2_t {
#[inline(always)]
fn from(value: mask64x2<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<float32x4x2_t, S> for f32x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: float32x4x2_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<f32x8<S>> for float32x4x2_t {
#[inline(always)]
fn from(value: f32x8<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int8x16x2_t, S> for i8x32<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int8x16x2_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i8x32<S>> for int8x16x2_t {
#[inline(always)]
fn from(value: i8x32<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<uint8x16x2_t, S> for u8x32<S> {
#[inline(always)]
fn simd_from(simd: S, arch: uint8x16x2_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u8x32<S>> for uint8x16x2_t {
#[inline(always)]
fn from(value: u8x32<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int8x16x2_t, S> for mask8x32<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int8x16x2_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<mask8x32<S>> for int8x16x2_t {
#[inline(always)]
fn from(value: mask8x32<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int16x8x2_t, S> for i16x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int16x8x2_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i16x16<S>> for int16x8x2_t {
#[inline(always)]
fn from(value: i16x16<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<uint16x8x2_t, S> for u16x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: uint16x8x2_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u16x16<S>> for uint16x8x2_t {
#[inline(always)]
fn from(value: u16x16<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int16x8x2_t, S> for mask16x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int16x8x2_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<mask16x16<S>> for int16x8x2_t {
#[inline(always)]
fn from(value: mask16x16<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int32x4x2_t, S> for i32x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int32x4x2_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i32x8<S>> for int32x4x2_t {
#[inline(always)]
fn from(value: i32x8<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<uint32x4x2_t, S> for u32x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: uint32x4x2_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u32x8<S>> for uint32x4x2_t {
#[inline(always)]
fn from(value: u32x8<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int32x4x2_t, S> for mask32x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int32x4x2_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<mask32x8<S>> for int32x4x2_t {
#[inline(always)]
fn from(value: mask32x8<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<float64x2x2_t, S> for f64x4<S> {
#[inline(always)]
fn simd_from(simd: S, arch: float64x2x2_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<f64x4<S>> for float64x2x2_t {
#[inline(always)]
fn from(value: f64x4<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int64x2x2_t, S> for i64x4<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int64x2x2_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i64x4<S>> for int64x2x2_t {
#[inline(always)]
fn from(value: i64x4<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<uint64x2x2_t, S> for u64x4<S> {
#[inline(always)]
fn simd_from(simd: S, arch: uint64x2x2_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u64x4<S>> for uint64x2x2_t {
#[inline(always)]
fn from(value: u64x4<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int64x2x2_t, S> for mask64x4<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int64x2x2_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<mask64x4<S>> for int64x2x2_t {
#[inline(always)]
fn from(value: mask64x4<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<float32x4x4_t, S> for f32x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: float32x4x4_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<f32x16<S>> for float32x4x4_t {
#[inline(always)]
fn from(value: f32x16<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int8x16x4_t, S> for i8x64<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int8x16x4_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i8x64<S>> for int8x16x4_t {
#[inline(always)]
fn from(value: i8x64<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<uint8x16x4_t, S> for u8x64<S> {
#[inline(always)]
fn simd_from(simd: S, arch: uint8x16x4_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u8x64<S>> for uint8x16x4_t {
#[inline(always)]
fn from(value: u8x64<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int8x16x4_t, S> for mask8x64<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int8x16x4_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<mask8x64<S>> for int8x16x4_t {
#[inline(always)]
fn from(value: mask8x64<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int16x8x4_t, S> for i16x32<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int16x8x4_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i16x32<S>> for int16x8x4_t {
#[inline(always)]
fn from(value: i16x32<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<uint16x8x4_t, S> for u16x32<S> {
#[inline(always)]
fn simd_from(simd: S, arch: uint16x8x4_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u16x32<S>> for uint16x8x4_t {
#[inline(always)]
fn from(value: u16x32<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int16x8x4_t, S> for mask16x32<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int16x8x4_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<mask16x32<S>> for int16x8x4_t {
#[inline(always)]
fn from(value: mask16x32<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int32x4x4_t, S> for i32x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int32x4x4_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i32x16<S>> for int32x4x4_t {
#[inline(always)]
fn from(value: i32x16<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<uint32x4x4_t, S> for u32x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: uint32x4x4_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u32x16<S>> for uint32x4x4_t {
#[inline(always)]
fn from(value: u32x16<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int32x4x4_t, S> for mask32x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int32x4x4_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<mask32x16<S>> for int32x4x4_t {
#[inline(always)]
fn from(value: mask32x16<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<float64x2x4_t, S> for f64x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: float64x2x4_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<f64x8<S>> for float64x2x4_t {
#[inline(always)]
fn from(value: f64x8<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int64x2x4_t, S> for i64x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int64x2x4_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i64x8<S>> for int64x2x4_t {
#[inline(always)]
fn from(value: i64x8<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<uint64x2x4_t, S> for u64x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: uint64x2x4_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u64x8<S>> for uint64x2x4_t {
#[inline(always)]
fn from(value: u64x8<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<int64x2x4_t, S> for mask64x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: int64x2x4_t) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<mask64x8<S>> for int64x2x4_t {
#[inline(always)]
fn from(value: mask64x8<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
crate::kernel!(
#[doc = r" This is a version of the `vext` intrinsic that takes a non-const shift argument. The shift is still"]
#[doc = r" expected to be constant in practice, so the match statement will be optimized out. This exists because"]
#[doc = r" Rust doesn't currently let you do math on const generics."]
#[inline(always)]
fn dyn_vext_128(neon: Neon, a: uint8x16_t, b: uint8x16_t, shift: usize) -> uint8x16_t {
match shift {
0usize => vextq_u8::<0i32>(a, b),
1usize => vextq_u8::<1i32>(a, b),
2usize => vextq_u8::<2i32>(a, b),
3usize => vextq_u8::<3i32>(a, b),
4usize => vextq_u8::<4i32>(a, b),
5usize => vextq_u8::<5i32>(a, b),
6usize => vextq_u8::<6i32>(a, b),
7usize => vextq_u8::<7i32>(a, b),
8usize => vextq_u8::<8i32>(a, b),
9usize => vextq_u8::<9i32>(a, b),
10usize => vextq_u8::<10i32>(a, b),
11usize => vextq_u8::<11i32>(a, b),
12usize => vextq_u8::<12i32>(a, b),
13usize => vextq_u8::<13i32>(a, b),
14usize => vextq_u8::<14i32>(a, b),
15usize => vextq_u8::<15i32>(a, b),
_ => unreachable!(),
}
}
);