use crate::{Level, arch_types::ArchTypes, prelude::*, seal::Seal};
use crate::{
f32x4, f32x8, f32x16, f64x2, f64x4, f64x8, i8x16, i8x32, i8x64, i16x8, i16x16, i16x32, i32x4,
i32x8, i32x16, i64x2, i64x4, i64x8, mask8x16, mask8x32, mask8x64, mask16x8, mask16x16,
mask16x32, mask32x4, mask32x8, mask32x16, mask64x2, mask64x4, mask64x8, u8x16, u8x32, u8x64,
u16x8, u16x16, u16x32, u32x4, u32x8, u32x16, u64x2, u64x4, u64x8,
};
use core::arch::wasm32::*;
use core::ops::*;
#[doc = "A token for WASM SIMD128, representing the \"wasm128\" level."]
#[doc = "# Browsing the documentation"]
#[doc = "The method list on this struct is very verbose."]
#[doc = "A better way to browse the docs is by looking at vector types such as [`u32x4`], [`f32x4`] or [`mask32x4`]."]
#[doc = "They include all the operations listed here, and also provide some additional convenience methods."]
#[derive(Clone, Copy, Debug)]
pub struct WasmSimd128 {
_private: (),
}
impl WasmSimd128 {
#[doc = r" Create a SIMD token proving that the WebAssembly `simd128` feature is enabled."]
#[doc = r""]
#[doc = r" WebAssembly does not support runtime feature detection, so this function is only"]
#[doc = r" available when the library is compiled with `simd128` enabled."]
#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))]
#[inline]
pub const fn assume_supported() -> Self {
Self { _private: () }
}
}
impl Seal for WasmSimd128 {}
impl ArchTypes for WasmSimd128 {
type f32x4 = crate::support::Aligned128<v128>;
type i8x16 = crate::support::Aligned128<v128>;
type u8x16 = crate::support::Aligned128<v128>;
type mask8x16 = crate::support::Aligned128<v128>;
type i16x8 = crate::support::Aligned128<v128>;
type u16x8 = crate::support::Aligned128<v128>;
type mask16x8 = crate::support::Aligned128<v128>;
type i32x4 = crate::support::Aligned128<v128>;
type u32x4 = crate::support::Aligned128<v128>;
type mask32x4 = crate::support::Aligned128<v128>;
type f64x2 = crate::support::Aligned128<v128>;
type i64x2 = crate::support::Aligned128<v128>;
type u64x2 = crate::support::Aligned128<v128>;
type mask64x2 = crate::support::Aligned128<v128>;
type f32x8 = crate::support::Aligned256<[v128; 2usize]>;
type i8x32 = crate::support::Aligned256<[v128; 2usize]>;
type u8x32 = crate::support::Aligned256<[v128; 2usize]>;
type mask8x32 = crate::support::Aligned256<[v128; 2usize]>;
type i16x16 = crate::support::Aligned256<[v128; 2usize]>;
type u16x16 = crate::support::Aligned256<[v128; 2usize]>;
type mask16x16 = crate::support::Aligned256<[v128; 2usize]>;
type i32x8 = crate::support::Aligned256<[v128; 2usize]>;
type u32x8 = crate::support::Aligned256<[v128; 2usize]>;
type mask32x8 = crate::support::Aligned256<[v128; 2usize]>;
type f64x4 = crate::support::Aligned256<[v128; 2usize]>;
type i64x4 = crate::support::Aligned256<[v128; 2usize]>;
type u64x4 = crate::support::Aligned256<[v128; 2usize]>;
type mask64x4 = crate::support::Aligned256<[v128; 2usize]>;
type f32x16 = crate::support::Aligned512<[v128; 4usize]>;
type i8x64 = crate::support::Aligned512<[v128; 4usize]>;
type u8x64 = crate::support::Aligned512<[v128; 4usize]>;
type mask8x64 = crate::support::Aligned512<[v128; 4usize]>;
type i16x32 = crate::support::Aligned512<[v128; 4usize]>;
type u16x32 = crate::support::Aligned512<[v128; 4usize]>;
type mask16x32 = crate::support::Aligned512<[v128; 4usize]>;
type i32x16 = crate::support::Aligned512<[v128; 4usize]>;
type u32x16 = crate::support::Aligned512<[v128; 4usize]>;
type mask32x16 = crate::support::Aligned512<[v128; 4usize]>;
type f64x8 = crate::support::Aligned512<[v128; 4usize]>;
type i64x8 = crate::support::Aligned512<[v128; 4usize]>;
type u64x8 = crate::support::Aligned512<[v128; 4usize]>;
type mask64x8 = crate::support::Aligned512<[v128; 4usize]>;
}
impl Simd for WasmSimd128 {
type f32s = f32x4<Self>;
type f64s = f64x2<Self>;
type u8s = u8x16<Self>;
type i8s = i8x16<Self>;
type u16s = u16x8<Self>;
type i16s = i16x8<Self>;
type u32s = u32x4<Self>;
type i32s = i32x4<Self>;
type u64s = u64x2<Self>;
type i64s = i64x2<Self>;
type mask8s = mask8x16<Self>;
type mask16s = mask16x8<Self>;
type mask32s = mask32x4<Self>;
type mask64s = mask64x2<Self>;
#[inline(always)]
fn level(self) -> Level {
Level::WasmSimd128(self)
}
#[inline]
fn vectorize<F: FnOnce() -> R, R>(self, f: F) -> R {
fn vectorize_inner<F: FnOnce() -> R, R>(f: F) -> R {
f()
}
vectorize_inner(f)
}
#[inline(always)]
fn splat_f32x4(self, val: f32) -> f32x4<Self> {
f32x4_splat(val).simd_into(self)
}
#[inline(always)]
fn slide_f32x4<const SHIFT: usize>(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = dyn_slide_128(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn abs_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
f32x4_abs(a.into()).simd_into(self)
}
#[inline(always)]
fn neg_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
f32x4_neg(a.into()).simd_into(self)
}
#[inline(always)]
fn sqrt_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
f32x4_sqrt(a.into()).simd_into(self)
}
#[inline(always)]
fn approximate_recip_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
1.0 / a
}
#[inline(always)]
fn add_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
f32x4_add(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn sub_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
f32x4_sub(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn mul_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
f32x4_mul(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn div_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
f32x4_div(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn copysign_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
let sign_mask = f32x4_splat(-0.0_f32);
let sign_bits = v128_and(b.into(), sign_mask.into());
let magnitude = v128_andnot(a.into(), sign_mask.into());
v128_or(magnitude, sign_bits).simd_into(self)
}
#[inline(always)]
fn max_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
f32x4_relaxed_max(a.into(), b.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
f32x4_max(a.into(), b.into()).simd_into(self)
}
}
#[inline(always)]
fn min_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
f32x4_relaxed_min(a.into(), b.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
f32x4_min(a.into(), b.into()).simd_into(self)
}
}
#[inline(always)]
fn max_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
let intermediate = f32x4_pmax(b.into(), a.into());
let b_is_nan = f32x4_ne(b.into(), b.into());
v128_bitselect(a.into(), intermediate, b_is_nan).simd_into(self)
}
#[inline(always)]
fn min_precise_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
let intermediate = f32x4_pmin(b.into(), a.into());
let b_is_nan = f32x4_ne(b.into(), b.into());
v128_bitselect(a.into(), intermediate, b_is_nan).simd_into(self)
}
#[inline(always)]
fn simd_eq_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
f32x4_eq(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_lt_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
f32x4_lt(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_le_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> mask32x4<Self> {
f32x4_le(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn zip_low_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
u32x4_shuffle::<0, 4, 1, 5>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn zip_high_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
u32x4_shuffle::<2, 6, 3, 7>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn unzip_low_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
u32x4_shuffle::<0, 2, 4, 6>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn unzip_high_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x4<Self> {
u32x4_shuffle::<1, 3, 5, 7>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn interleave_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> (f32x4<Self>, f32x4<Self>) {
(self.zip_low_f32x4(a, b), self.zip_high_f32x4(a, b))
}
#[inline(always)]
fn deinterleave_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> (f32x4<Self>, f32x4<Self>) {
(self.unzip_low_f32x4(a, b), self.unzip_high_f32x4(a, b))
}
#[inline(always)]
fn mul_add_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
f32x4_relaxed_madd(a.into(), b.into(), c.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
self.add_f32x4(self.mul_f32x4(a, b), c)
}
}
#[inline(always)]
fn mul_sub_f32x4(self, a: f32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
f32x4_relaxed_madd(a.into(), b.into(), f32x4_neg(c.into())).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
self.sub_f32x4(self.mul_f32x4(a, b), c)
}
}
#[inline(always)]
fn floor_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
f32x4_floor(a.into()).simd_into(self)
}
#[inline(always)]
fn ceil_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
f32x4_ceil(a.into()).simd_into(self)
}
#[inline(always)]
fn round_ties_even_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
f32x4_nearest(a.into()).simd_into(self)
}
#[inline(always)]
fn fract_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
self.sub_f32x4(a, self.trunc_f32x4(a))
}
#[inline(always)]
fn trunc_f32x4(self, a: f32x4<Self>) -> f32x4<Self> {
f32x4_trunc(a.into()).simd_into(self)
}
#[inline(always)]
fn select_f32x4(self, a: mask32x4<Self>, b: f32x4<Self>, c: f32x4<Self>) -> f32x4<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
i32x4_relaxed_laneselect(b.into(), c.into(), a.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
v128_bitselect(b.into(), c.into(), a.into()).simd_into(self)
}
}
#[inline(always)]
fn combine_f32x4(self, a: f32x4<Self>, b: f32x4<Self>) -> f32x8<Self> {
f32x8 {
val: crate::support::Aligned256([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn load_four_interleaved_f32x4(self, src: &[f32; 16usize]) -> [f32x4<Self>; 4usize] {
let (chunks, []) = src.as_chunks::<4usize>() else {
unreachable!()
};
let v0: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[0]);
let v1: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[1]);
let v2: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[2]);
let v3: v128 = crate::transmute::checked_transmute_copy::<[f32; 4usize], v128>(&chunks[3]);
let v01_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v1);
let v23_lower = u32x4_shuffle::<0, 4, 1, 5>(v2, v3);
let v01_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v1);
let v23_upper = u32x4_shuffle::<2, 6, 3, 7>(v2, v3);
let out0 = u32x4_shuffle::<0, 1, 4, 5>(v01_lower, v23_lower);
let out1 = u32x4_shuffle::<2, 3, 6, 7>(v01_lower, v23_lower);
let out2 = u32x4_shuffle::<0, 1, 4, 5>(v01_upper, v23_upper);
let out3 = u32x4_shuffle::<2, 3, 6, 7>(v01_upper, v23_upper);
[
out0.simd_into(self),
out1.simd_into(self),
out2.simd_into(self),
out3.simd_into(self),
]
}
#[inline(always)]
fn store_four_interleaved_f32x4(
self,
vectors: [f32x4<Self>; 4usize],
dest: &mut [f32; 16usize],
) -> () {
let v0: v128 = vectors[0].into();
let v1: v128 = vectors[1].into();
let v2: v128 = vectors[2].into();
let v3: v128 = vectors[3].into();
let v02_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v2);
let v13_lower = u32x4_shuffle::<0, 4, 1, 5>(v1, v3);
let v02_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v2);
let v13_upper = u32x4_shuffle::<2, 6, 3, 7>(v1, v3);
let out0 = u32x4_shuffle::<0, 4, 1, 5>(v02_lower, v13_lower);
let out1 = u32x4_shuffle::<2, 6, 3, 7>(v02_lower, v13_lower);
let out2 = u32x4_shuffle::<0, 4, 1, 5>(v02_upper, v13_upper);
let out3 = u32x4_shuffle::<2, 6, 3, 7>(v02_upper, v13_upper);
let (chunks, []) = dest.as_chunks_mut::<4usize>() else {
unreachable!()
};
crate::transmute::checked_transmute_store::<v128, [f32; 4usize]>(out0, &mut chunks[0]);
crate::transmute::checked_transmute_store::<v128, [f32; 4usize]>(out1, &mut chunks[1]);
crate::transmute::checked_transmute_store::<v128, [f32; 4usize]>(out2, &mut chunks[2]);
crate::transmute::checked_transmute_store::<v128, [f32; 4usize]>(out3, &mut chunks[3]);
}
#[inline(always)]
fn widen_f32x4(self, a: f32x4<Self>) -> (f64x2<Self>, f64x2<Self>) {
let a = a.into();
(
f64x2_promote_low_f32x4(a).simd_into(self),
f64x2_promote_low_f32x4(i64x2_shuffle::<1, 1>(a, a)).simd_into(self),
)
}
#[inline(always)]
fn cvt_u32_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
u32x4_relaxed_trunc_f32x4(a.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
self.cvt_u32_precise_f32x4(a)
}
}
#[inline(always)]
fn cvt_u32_precise_f32x4(self, a: f32x4<Self>) -> u32x4<Self> {
u32x4_trunc_sat_f32x4(a.into()).simd_into(self)
}
#[inline(always)]
fn cvt_i32_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
i32x4_relaxed_trunc_f32x4(a.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
self.cvt_i32_precise_f32x4(a)
}
}
#[inline(always)]
fn cvt_i32_precise_f32x4(self, a: f32x4<Self>) -> i32x4<Self> {
i32x4_trunc_sat_f32x4(a.into()).simd_into(self)
}
#[inline(always)]
fn splat_i8x16(self, val: i8) -> i8x16<Self> {
i8x16_splat(val).simd_into(self)
}
#[inline(always)]
fn slide_i8x16<const SHIFT: usize>(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = dyn_slide_128(Bytes::to_bytes(a).val.0, Bytes::to_bytes(b).val.0, SHIFT);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
i8x16_add(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn sub_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
i8x16_sub(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn mul_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
let low = i16x8_extmul_low_i8x16(a.into(), b.into());
let high = i16x8_extmul_high_i8x16(a.into(), b.into());
u8x16_shuffle::<0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30>(low, high)
.simd_into(self)
}
#[inline(always)]
fn and_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
v128_and(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn or_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
v128_or(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn xor_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
v128_xor(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn not_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
v128_not(a.into()).simd_into(self)
}
#[inline(always)]
fn shl_i8x16(self, a: i8x16<Self>, shift: u32) -> i8x16<Self> {
i8x16_shl(a.into(), shift).simd_into(self)
}
#[inline(always)]
fn shlv_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
[
i8::wrapping_shl(a[0usize], b[0usize] as u32),
i8::wrapping_shl(a[1usize], b[1usize] as u32),
i8::wrapping_shl(a[2usize], b[2usize] as u32),
i8::wrapping_shl(a[3usize], b[3usize] as u32),
i8::wrapping_shl(a[4usize], b[4usize] as u32),
i8::wrapping_shl(a[5usize], b[5usize] as u32),
i8::wrapping_shl(a[6usize], b[6usize] as u32),
i8::wrapping_shl(a[7usize], b[7usize] as u32),
i8::wrapping_shl(a[8usize], b[8usize] as u32),
i8::wrapping_shl(a[9usize], b[9usize] as u32),
i8::wrapping_shl(a[10usize], b[10usize] as u32),
i8::wrapping_shl(a[11usize], b[11usize] as u32),
i8::wrapping_shl(a[12usize], b[12usize] as u32),
i8::wrapping_shl(a[13usize], b[13usize] as u32),
i8::wrapping_shl(a[14usize], b[14usize] as u32),
i8::wrapping_shl(a[15usize], b[15usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn shr_i8x16(self, a: i8x16<Self>, shift: u32) -> i8x16<Self> {
i8x16_shr(a.into(), shift).simd_into(self)
}
#[inline(always)]
fn shrv_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
[
i8::wrapping_shr(a[0usize], b[0usize] as u32),
i8::wrapping_shr(a[1usize], b[1usize] as u32),
i8::wrapping_shr(a[2usize], b[2usize] as u32),
i8::wrapping_shr(a[3usize], b[3usize] as u32),
i8::wrapping_shr(a[4usize], b[4usize] as u32),
i8::wrapping_shr(a[5usize], b[5usize] as u32),
i8::wrapping_shr(a[6usize], b[6usize] as u32),
i8::wrapping_shr(a[7usize], b[7usize] as u32),
i8::wrapping_shr(a[8usize], b[8usize] as u32),
i8::wrapping_shr(a[9usize], b[9usize] as u32),
i8::wrapping_shr(a[10usize], b[10usize] as u32),
i8::wrapping_shr(a[11usize], b[11usize] as u32),
i8::wrapping_shr(a[12usize], b[12usize] as u32),
i8::wrapping_shr(a[13usize], b[13usize] as u32),
i8::wrapping_shr(a[14usize], b[14usize] as u32),
i8::wrapping_shr(a[15usize], b[15usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn max_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
i8x16_max(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn min_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
i8x16_min(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_eq_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
i8x16_eq(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_lt_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
i8x16_lt(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_le_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> mask8x16<Self> {
i8x16_le(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn zip_low_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(a.into(), b.into())
.simd_into(self)
}
#[inline(always)]
fn zip_high_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(
a.into(),
b.into(),
)
.simd_into(self)
}
#[inline(always)]
fn unzip_low_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
u8x16_shuffle::<0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30>(
a.into(),
b.into(),
)
.simd_into(self)
}
#[inline(always)]
fn unzip_high_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x16<Self> {
u8x16_shuffle::<1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31>(
a.into(),
b.into(),
)
.simd_into(self)
}
#[inline(always)]
fn interleave_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i8x16<Self>, i8x16<Self>) {
(self.zip_low_i8x16(a, b), self.zip_high_i8x16(a, b))
}
#[inline(always)]
fn deinterleave_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> (i8x16<Self>, i8x16<Self>) {
(self.unzip_low_i8x16(a, b), self.unzip_high_i8x16(a, b))
}
#[inline(always)]
fn select_i8x16(self, a: mask8x16<Self>, b: i8x16<Self>, c: i8x16<Self>) -> i8x16<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
i8x16_relaxed_laneselect(b.into(), c.into(), a.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
v128_bitselect(b.into(), c.into(), a.into()).simd_into(self)
}
}
#[inline(always)]
fn combine_i8x16(self, a: i8x16<Self>, b: i8x16<Self>) -> i8x32<Self> {
i8x32 {
val: crate::support::Aligned256([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn neg_i8x16(self, a: i8x16<Self>) -> i8x16<Self> {
i8x16_neg(a.into()).simd_into(self)
}
#[inline(always)]
fn load_four_interleaved_i8x16(self, src: &[i8; 64usize]) -> [i8x16<Self>; 4usize] {
let (chunks, []) = src.as_chunks::<16usize>() else {
unreachable!()
};
let v0: v128 = crate::transmute::checked_transmute_copy::<[i8; 16usize], v128>(&chunks[0]);
let v1: v128 = crate::transmute::checked_transmute_copy::<[i8; 16usize], v128>(&chunks[1]);
let v2: v128 = crate::transmute::checked_transmute_copy::<[i8; 16usize], v128>(&chunks[2]);
let v3: v128 = crate::transmute::checked_transmute_copy::<[i8; 16usize], v128>(&chunks[3]);
let v01_lower =
u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v0, v1);
let v23_lower =
u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v2, v3);
let v01_upper =
u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v0, v1);
let v23_upper =
u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v2, v3);
let out0 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>(
v01_lower, v23_lower,
);
let out1 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>(
v01_lower, v23_lower,
);
let out2 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>(
v01_upper, v23_upper,
);
let out3 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>(
v01_upper, v23_upper,
);
[
out0.simd_into(self),
out1.simd_into(self),
out2.simd_into(self),
out3.simd_into(self),
]
}
#[inline(always)]
fn store_four_interleaved_i8x16(
self,
vectors: [i8x16<Self>; 4usize],
dest: &mut [i8; 64usize],
) -> () {
let v0: v128 = vectors[0].into();
let v1: v128 = vectors[1].into();
let v2: v128 = vectors[2].into();
let v3: v128 = vectors[3].into();
let v02_lower =
u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v0, v2);
let v13_lower =
u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v1, v3);
let v02_upper =
u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v0, v2);
let v13_upper =
u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v1, v3);
let out0 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(
v02_lower, v13_lower,
);
let out1 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(
v02_lower, v13_lower,
);
let out2 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(
v02_upper, v13_upper,
);
let out3 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(
v02_upper, v13_upper,
);
let (chunks, []) = dest.as_chunks_mut::<16usize>() else {
unreachable!()
};
crate::transmute::checked_transmute_store::<v128, [i8; 16usize]>(out0, &mut chunks[0]);
crate::transmute::checked_transmute_store::<v128, [i8; 16usize]>(out1, &mut chunks[1]);
crate::transmute::checked_transmute_store::<v128, [i8; 16usize]>(out2, &mut chunks[2]);
crate::transmute::checked_transmute_store::<v128, [i8; 16usize]>(out3, &mut chunks[3]);
}
#[inline(always)]
fn widen_i8x16(self, a: i8x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
(
i16x8_extend_low_i8x16(a.into()).simd_into(self),
i16x8_extend_high_i8x16(a.into()).simd_into(self),
)
}
#[inline(always)]
fn splat_u8x16(self, val: u8) -> u8x16<Self> {
u8x16_splat(val).simd_into(self)
}
#[inline(always)]
fn slide_u8x16<const SHIFT: usize>(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = dyn_slide_128(Bytes::to_bytes(a).val.0, Bytes::to_bytes(b).val.0, SHIFT);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into());
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
self.swizzle_dyn_precise_u8x16(a, indices)
}
#[inline(always)]
fn swizzle_dyn_precise_u8x16(self, a: u8x16<Self>, indices: u8x16<Self>) -> u8x16<Self> {
let result = u8x16_swizzle(Bytes::to_bytes(a).val.0, indices.into());
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
u8x16_add(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn sub_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
u8x16_sub(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn mul_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
let low = u16x8_extmul_low_u8x16(a.into(), b.into());
let high = u16x8_extmul_high_u8x16(a.into(), b.into());
u8x16_shuffle::<0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30>(low, high)
.simd_into(self)
}
#[inline(always)]
fn and_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
v128_and(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn or_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
v128_or(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn xor_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
v128_xor(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn not_u8x16(self, a: u8x16<Self>) -> u8x16<Self> {
v128_not(a.into()).simd_into(self)
}
#[inline(always)]
fn shl_u8x16(self, a: u8x16<Self>, shift: u32) -> u8x16<Self> {
u8x16_shl(a.into(), shift).simd_into(self)
}
#[inline(always)]
fn shlv_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
[
u8::wrapping_shl(a[0usize], b[0usize] as u32),
u8::wrapping_shl(a[1usize], b[1usize] as u32),
u8::wrapping_shl(a[2usize], b[2usize] as u32),
u8::wrapping_shl(a[3usize], b[3usize] as u32),
u8::wrapping_shl(a[4usize], b[4usize] as u32),
u8::wrapping_shl(a[5usize], b[5usize] as u32),
u8::wrapping_shl(a[6usize], b[6usize] as u32),
u8::wrapping_shl(a[7usize], b[7usize] as u32),
u8::wrapping_shl(a[8usize], b[8usize] as u32),
u8::wrapping_shl(a[9usize], b[9usize] as u32),
u8::wrapping_shl(a[10usize], b[10usize] as u32),
u8::wrapping_shl(a[11usize], b[11usize] as u32),
u8::wrapping_shl(a[12usize], b[12usize] as u32),
u8::wrapping_shl(a[13usize], b[13usize] as u32),
u8::wrapping_shl(a[14usize], b[14usize] as u32),
u8::wrapping_shl(a[15usize], b[15usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn shr_u8x16(self, a: u8x16<Self>, shift: u32) -> u8x16<Self> {
u8x16_shr(a.into(), shift).simd_into(self)
}
#[inline(always)]
fn shrv_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
[
u8::wrapping_shr(a[0usize], b[0usize] as u32),
u8::wrapping_shr(a[1usize], b[1usize] as u32),
u8::wrapping_shr(a[2usize], b[2usize] as u32),
u8::wrapping_shr(a[3usize], b[3usize] as u32),
u8::wrapping_shr(a[4usize], b[4usize] as u32),
u8::wrapping_shr(a[5usize], b[5usize] as u32),
u8::wrapping_shr(a[6usize], b[6usize] as u32),
u8::wrapping_shr(a[7usize], b[7usize] as u32),
u8::wrapping_shr(a[8usize], b[8usize] as u32),
u8::wrapping_shr(a[9usize], b[9usize] as u32),
u8::wrapping_shr(a[10usize], b[10usize] as u32),
u8::wrapping_shr(a[11usize], b[11usize] as u32),
u8::wrapping_shr(a[12usize], b[12usize] as u32),
u8::wrapping_shr(a[13usize], b[13usize] as u32),
u8::wrapping_shr(a[14usize], b[14usize] as u32),
u8::wrapping_shr(a[15usize], b[15usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn max_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
u8x16_max(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn min_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
u8x16_min(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_eq_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
u8x16_eq(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_lt_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
u8x16_lt(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_le_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> mask8x16<Self> {
u8x16_le(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn zip_low_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(a.into(), b.into())
.simd_into(self)
}
#[inline(always)]
fn zip_high_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(
a.into(),
b.into(),
)
.simd_into(self)
}
#[inline(always)]
fn unzip_low_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
u8x16_shuffle::<0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30>(
a.into(),
b.into(),
)
.simd_into(self)
}
#[inline(always)]
fn unzip_high_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x16<Self> {
u8x16_shuffle::<1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31>(
a.into(),
b.into(),
)
.simd_into(self)
}
#[inline(always)]
fn interleave_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u8x16<Self>, u8x16<Self>) {
(self.zip_low_u8x16(a, b), self.zip_high_u8x16(a, b))
}
#[inline(always)]
fn deinterleave_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> (u8x16<Self>, u8x16<Self>) {
(self.unzip_low_u8x16(a, b), self.unzip_high_u8x16(a, b))
}
#[inline(always)]
fn select_u8x16(self, a: mask8x16<Self>, b: u8x16<Self>, c: u8x16<Self>) -> u8x16<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
i8x16_relaxed_laneselect(b.into(), c.into(), a.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
v128_bitselect(b.into(), c.into(), a.into()).simd_into(self)
}
}
#[inline(always)]
fn combine_u8x16(self, a: u8x16<Self>, b: u8x16<Self>) -> u8x32<Self> {
u8x32 {
val: crate::support::Aligned256([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn load_four_interleaved_u8x16(self, src: &[u8; 64usize]) -> [u8x16<Self>; 4usize] {
let (chunks, []) = src.as_chunks::<16usize>() else {
unreachable!()
};
let v0: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[0]);
let v1: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[1]);
let v2: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[2]);
let v3: v128 = crate::transmute::checked_transmute_copy::<[u8; 16usize], v128>(&chunks[3]);
let v01_lower =
u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v0, v1);
let v23_lower =
u8x16_shuffle::<0, 4, 8, 12, 16, 20, 24, 28, 1, 5, 9, 13, 17, 21, 25, 29>(v2, v3);
let v01_upper =
u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v0, v1);
let v23_upper =
u8x16_shuffle::<2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31>(v2, v3);
let out0 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>(
v01_lower, v23_lower,
);
let out1 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>(
v01_lower, v23_lower,
);
let out2 = u8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23>(
v01_upper, v23_upper,
);
let out3 = u8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31>(
v01_upper, v23_upper,
);
[
out0.simd_into(self),
out1.simd_into(self),
out2.simd_into(self),
out3.simd_into(self),
]
}
#[inline(always)]
fn store_four_interleaved_u8x16(
self,
vectors: [u8x16<Self>; 4usize],
dest: &mut [u8; 64usize],
) -> () {
let v0: v128 = vectors[0].into();
let v1: v128 = vectors[1].into();
let v2: v128 = vectors[2].into();
let v3: v128 = vectors[3].into();
let v02_lower =
u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v0, v2);
let v13_lower =
u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(v1, v3);
let v02_upper =
u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v0, v2);
let v13_upper =
u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(v1, v3);
let out0 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(
v02_lower, v13_lower,
);
let out1 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(
v02_lower, v13_lower,
);
let out2 = u8x16_shuffle::<0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23>(
v02_upper, v13_upper,
);
let out3 = u8x16_shuffle::<8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31>(
v02_upper, v13_upper,
);
let (chunks, []) = dest.as_chunks_mut::<16usize>() else {
unreachable!()
};
crate::transmute::checked_transmute_store::<v128, [u8; 16usize]>(out0, &mut chunks[0]);
crate::transmute::checked_transmute_store::<v128, [u8; 16usize]>(out1, &mut chunks[1]);
crate::transmute::checked_transmute_store::<v128, [u8; 16usize]>(out2, &mut chunks[2]);
crate::transmute::checked_transmute_store::<v128, [u8; 16usize]>(out3, &mut chunks[3]);
}
#[inline(always)]
fn widen_u8x16(self, a: u8x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
(
i16x8_extend_low_u8x16(a.into()).simd_into(self),
i16x8_extend_high_u8x16(a.into()).simd_into(self),
)
}
#[inline(always)]
fn splat_mask8x16(self, val: bool) -> mask8x16<Self> {
let val: i8 = if val { !0 } else { 0 };
i8x16_splat(val).simd_into(self)
}
#[inline(always)]
fn from_bitmask_mask8x16(self, bits: u64) -> mask8x16<Self> {
let lo = i8x16_splat(bits as i8);
let hi = i8x16_splat((bits >> 8) as i8);
let bytes = u8x16_shuffle::<0, 0, 0, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16, 16>(lo, hi);
let powers = u8x16(1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128);
let selected = v128_and(bytes, powers);
i8x16_ne(selected, i8x16_splat(0)).simd_into(self)
}
#[inline(always)]
fn to_bitmask_mask8x16(self, a: mask8x16<Self>) -> u64 {
i8x16_bitmask(a.into()) as u64
}
#[inline(always)]
fn set_mask8x16(self, a: &mut mask8x16<Self>, index: usize, value: bool) -> () {
assert!(
index < 16usize,
"mask lane index {index} is out of bounds for {} lanes",
16usize
);
let mut lanes: [i8; 16usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn and_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
v128_and(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn or_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
v128_or(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn xor_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
v128_xor(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn not_mask8x16(self, a: mask8x16<Self>) -> mask8x16<Self> {
v128_not(a.into()).simd_into(self)
}
#[inline(always)]
fn select_mask8x16(
self,
a: mask8x16<Self>,
b: mask8x16<Self>,
c: mask8x16<Self>,
) -> mask8x16<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
i8x16_relaxed_laneselect(b.into(), c.into(), a.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
v128_bitselect(b.into(), c.into(), a.into()).simd_into(self)
}
}
#[inline(always)]
fn simd_eq_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x16<Self> {
i8x16_eq(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn any_true_mask8x16(self, a: mask8x16<Self>) -> bool {
v128_any_true(a.into())
}
#[inline(always)]
fn all_true_mask8x16(self, a: mask8x16<Self>) -> bool {
i8x16_all_true(a.into())
}
#[inline(always)]
fn any_false_mask8x16(self, a: mask8x16<Self>) -> bool {
!i8x16_all_true(a.into())
}
#[inline(always)]
fn all_false_mask8x16(self, a: mask8x16<Self>) -> bool {
!v128_any_true(a.into())
}
#[inline(always)]
fn combine_mask8x16(self, a: mask8x16<Self>, b: mask8x16<Self>) -> mask8x32<Self> {
mask8x32 {
val: crate::support::Aligned256([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn splat_i16x8(self, val: i16) -> i16x8<Self> {
i16x8_splat(val).simd_into(self)
}
#[inline(always)]
fn slide_i16x8<const SHIFT: usize>(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = dyn_slide_128(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 2usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
i16x8_add(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn sub_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
i16x8_sub(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn mul_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
i16x8_mul(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn and_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
v128_and(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn or_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
v128_or(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn xor_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
v128_xor(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn not_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
v128_not(a.into()).simd_into(self)
}
#[inline(always)]
fn shl_i16x8(self, a: i16x8<Self>, shift: u32) -> i16x8<Self> {
i16x8_shl(a.into(), shift).simd_into(self)
}
#[inline(always)]
fn shlv_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
[
i16::wrapping_shl(a[0usize], b[0usize] as u32),
i16::wrapping_shl(a[1usize], b[1usize] as u32),
i16::wrapping_shl(a[2usize], b[2usize] as u32),
i16::wrapping_shl(a[3usize], b[3usize] as u32),
i16::wrapping_shl(a[4usize], b[4usize] as u32),
i16::wrapping_shl(a[5usize], b[5usize] as u32),
i16::wrapping_shl(a[6usize], b[6usize] as u32),
i16::wrapping_shl(a[7usize], b[7usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn shr_i16x8(self, a: i16x8<Self>, shift: u32) -> i16x8<Self> {
i16x8_shr(a.into(), shift).simd_into(self)
}
#[inline(always)]
fn shrv_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
[
i16::wrapping_shr(a[0usize], b[0usize] as u32),
i16::wrapping_shr(a[1usize], b[1usize] as u32),
i16::wrapping_shr(a[2usize], b[2usize] as u32),
i16::wrapping_shr(a[3usize], b[3usize] as u32),
i16::wrapping_shr(a[4usize], b[4usize] as u32),
i16::wrapping_shr(a[5usize], b[5usize] as u32),
i16::wrapping_shr(a[6usize], b[6usize] as u32),
i16::wrapping_shr(a[7usize], b[7usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn max_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
i16x8_max(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn min_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
i16x8_min(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_eq_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
i16x8_eq(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_lt_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
i16x8_lt(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_le_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> mask16x8<Self> {
i16x8_le(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn zip_low_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn zip_high_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn unzip_low_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
u16x8_shuffle::<0, 2, 4, 6, 8, 10, 12, 14>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn unzip_high_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x8<Self> {
u16x8_shuffle::<1, 3, 5, 7, 9, 11, 13, 15>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn interleave_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i16x8<Self>, i16x8<Self>) {
(self.zip_low_i16x8(a, b), self.zip_high_i16x8(a, b))
}
#[inline(always)]
fn deinterleave_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> (i16x8<Self>, i16x8<Self>) {
(self.unzip_low_i16x8(a, b), self.unzip_high_i16x8(a, b))
}
#[inline(always)]
fn select_i16x8(self, a: mask16x8<Self>, b: i16x8<Self>, c: i16x8<Self>) -> i16x8<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
i16x8_relaxed_laneselect(b.into(), c.into(), a.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
v128_bitselect(b.into(), c.into(), a.into()).simd_into(self)
}
}
#[inline(always)]
fn combine_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i16x16<Self> {
i16x16 {
val: crate::support::Aligned256([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn neg_i16x8(self, a: i16x8<Self>) -> i16x8<Self> {
i16x8_neg(a.into()).simd_into(self)
}
#[inline(always)]
fn load_four_interleaved_i16x8(self, src: &[i16; 32usize]) -> [i16x8<Self>; 4usize] {
let (chunks, []) = src.as_chunks::<8usize>() else {
unreachable!()
};
let v0: v128 = crate::transmute::checked_transmute_copy::<[i16; 8usize], v128>(&chunks[0]);
let v1: v128 = crate::transmute::checked_transmute_copy::<[i16; 8usize], v128>(&chunks[1]);
let v2: v128 = crate::transmute::checked_transmute_copy::<[i16; 8usize], v128>(&chunks[2]);
let v3: v128 = crate::transmute::checked_transmute_copy::<[i16; 8usize], v128>(&chunks[3]);
let v01_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v0, v1);
let v23_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v2, v3);
let v01_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v0, v1);
let v23_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v2, v3);
let out0 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_lower, v23_lower);
let out1 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_lower, v23_lower);
let out2 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_upper, v23_upper);
let out3 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_upper, v23_upper);
[
out0.simd_into(self),
out1.simd_into(self),
out2.simd_into(self),
out3.simd_into(self),
]
}
#[inline(always)]
fn store_four_interleaved_i16x8(
self,
vectors: [i16x8<Self>; 4usize],
dest: &mut [i16; 32usize],
) -> () {
let v0: v128 = vectors[0].into();
let v1: v128 = vectors[1].into();
let v2: v128 = vectors[2].into();
let v3: v128 = vectors[3].into();
let v02_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v0, v2);
let v13_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v1, v3);
let v02_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v0, v2);
let v13_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v1, v3);
let out0 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_lower, v13_lower);
let out1 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_lower, v13_lower);
let out2 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_upper, v13_upper);
let out3 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_upper, v13_upper);
let (chunks, []) = dest.as_chunks_mut::<8usize>() else {
unreachable!()
};
crate::transmute::checked_transmute_store::<v128, [i16; 8usize]>(out0, &mut chunks[0]);
crate::transmute::checked_transmute_store::<v128, [i16; 8usize]>(out1, &mut chunks[1]);
crate::transmute::checked_transmute_store::<v128, [i16; 8usize]>(out2, &mut chunks[2]);
crate::transmute::checked_transmute_store::<v128, [i16; 8usize]>(out3, &mut chunks[3]);
}
#[inline(always)]
fn widen_i16x8(self, a: i16x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
(
i32x4_extend_low_i16x8(a.into()).simd_into(self),
i32x4_extend_high_i16x8(a.into()).simd_into(self),
)
}
#[inline(always)]
fn narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
let mask = i16x8_splat(255);
u8x16_narrow_i16x8(v128_and(a.into(), mask), v128_and(b.into(), mask)).simd_into(self)
}
#[inline(always)]
fn saturating_narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
i8x16_narrow_i16x8(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn relaxed_narrow_i16x8(self, a: i16x8<Self>, b: i16x8<Self>) -> i8x16<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= i8::MIN as i16 && value <= i8::MAX as i16 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.saturating_narrow_i16x8(a, b)
}
#[inline(always)]
fn splat_u16x8(self, val: u16) -> u16x8<Self> {
u16x8_splat(val).simd_into(self)
}
#[inline(always)]
fn slide_u16x8<const SHIFT: usize>(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = dyn_slide_128(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 2usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
u16x8_add(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn sub_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
u16x8_sub(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn mul_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
u16x8_mul(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn and_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
v128_and(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn or_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
v128_or(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn xor_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
v128_xor(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn not_u16x8(self, a: u16x8<Self>) -> u16x8<Self> {
v128_not(a.into()).simd_into(self)
}
#[inline(always)]
fn shl_u16x8(self, a: u16x8<Self>, shift: u32) -> u16x8<Self> {
u16x8_shl(a.into(), shift).simd_into(self)
}
#[inline(always)]
fn shlv_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
[
u16::wrapping_shl(a[0usize], b[0usize] as u32),
u16::wrapping_shl(a[1usize], b[1usize] as u32),
u16::wrapping_shl(a[2usize], b[2usize] as u32),
u16::wrapping_shl(a[3usize], b[3usize] as u32),
u16::wrapping_shl(a[4usize], b[4usize] as u32),
u16::wrapping_shl(a[5usize], b[5usize] as u32),
u16::wrapping_shl(a[6usize], b[6usize] as u32),
u16::wrapping_shl(a[7usize], b[7usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn shr_u16x8(self, a: u16x8<Self>, shift: u32) -> u16x8<Self> {
u16x8_shr(a.into(), shift).simd_into(self)
}
#[inline(always)]
fn shrv_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
[
u16::wrapping_shr(a[0usize], b[0usize] as u32),
u16::wrapping_shr(a[1usize], b[1usize] as u32),
u16::wrapping_shr(a[2usize], b[2usize] as u32),
u16::wrapping_shr(a[3usize], b[3usize] as u32),
u16::wrapping_shr(a[4usize], b[4usize] as u32),
u16::wrapping_shr(a[5usize], b[5usize] as u32),
u16::wrapping_shr(a[6usize], b[6usize] as u32),
u16::wrapping_shr(a[7usize], b[7usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn max_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
u16x8_max(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn min_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
u16x8_min(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_eq_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
u16x8_eq(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_lt_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
u16x8_lt(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_le_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> mask16x8<Self> {
u16x8_le(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn zip_low_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn zip_high_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn unzip_low_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
u16x8_shuffle::<0, 2, 4, 6, 8, 10, 12, 14>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn unzip_high_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x8<Self> {
u16x8_shuffle::<1, 3, 5, 7, 9, 11, 13, 15>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn interleave_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u16x8<Self>, u16x8<Self>) {
(self.zip_low_u16x8(a, b), self.zip_high_u16x8(a, b))
}
#[inline(always)]
fn deinterleave_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> (u16x8<Self>, u16x8<Self>) {
(self.unzip_low_u16x8(a, b), self.unzip_high_u16x8(a, b))
}
#[inline(always)]
fn select_u16x8(self, a: mask16x8<Self>, b: u16x8<Self>, c: u16x8<Self>) -> u16x8<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
i16x8_relaxed_laneselect(b.into(), c.into(), a.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
v128_bitselect(b.into(), c.into(), a.into()).simd_into(self)
}
}
#[inline(always)]
fn combine_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u16x16<Self> {
u16x16 {
val: crate::support::Aligned256([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn load_four_interleaved_u16x8(self, src: &[u16; 32usize]) -> [u16x8<Self>; 4usize] {
let (chunks, []) = src.as_chunks::<8usize>() else {
unreachable!()
};
let v0: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[0]);
let v1: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[1]);
let v2: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[2]);
let v3: v128 = crate::transmute::checked_transmute_copy::<[u16; 8usize], v128>(&chunks[3]);
let v01_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v0, v1);
let v23_lower = u16x8_shuffle::<0, 4, 8, 12, 1, 5, 9, 13>(v2, v3);
let v01_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v0, v1);
let v23_upper = u16x8_shuffle::<2, 6, 10, 14, 3, 7, 11, 15>(v2, v3);
let out0 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_lower, v23_lower);
let out1 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_lower, v23_lower);
let out2 = u16x8_shuffle::<0, 1, 2, 3, 8, 9, 10, 11>(v01_upper, v23_upper);
let out3 = u16x8_shuffle::<4, 5, 6, 7, 12, 13, 14, 15>(v01_upper, v23_upper);
[
out0.simd_into(self),
out1.simd_into(self),
out2.simd_into(self),
out3.simd_into(self),
]
}
#[inline(always)]
fn store_four_interleaved_u16x8(
self,
vectors: [u16x8<Self>; 4usize],
dest: &mut [u16; 32usize],
) -> () {
let v0: v128 = vectors[0].into();
let v1: v128 = vectors[1].into();
let v2: v128 = vectors[2].into();
let v3: v128 = vectors[3].into();
let v02_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v0, v2);
let v13_lower = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v1, v3);
let v02_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v0, v2);
let v13_upper = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v1, v3);
let out0 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_lower, v13_lower);
let out1 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_lower, v13_lower);
let out2 = u16x8_shuffle::<0, 8, 1, 9, 2, 10, 3, 11>(v02_upper, v13_upper);
let out3 = u16x8_shuffle::<4, 12, 5, 13, 6, 14, 7, 15>(v02_upper, v13_upper);
let (chunks, []) = dest.as_chunks_mut::<8usize>() else {
unreachable!()
};
crate::transmute::checked_transmute_store::<v128, [u16; 8usize]>(out0, &mut chunks[0]);
crate::transmute::checked_transmute_store::<v128, [u16; 8usize]>(out1, &mut chunks[1]);
crate::transmute::checked_transmute_store::<v128, [u16; 8usize]>(out2, &mut chunks[2]);
crate::transmute::checked_transmute_store::<v128, [u16; 8usize]>(out3, &mut chunks[3]);
}
#[inline(always)]
fn widen_u16x8(self, a: u16x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
(
i32x4_extend_low_u16x8(a.into()).simd_into(self),
i32x4_extend_high_u16x8(a.into()).simd_into(self),
)
}
#[inline(always)]
fn narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
let mask = u16x8_splat(255);
u8x16_narrow_i16x8(v128_and(a.into(), mask), v128_and(b.into(), mask)).simd_into(self)
}
#[inline(always)]
fn saturating_narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
let max = u16x8_splat(255);
u8x16_narrow_i16x8(u16x8_min(a.into(), max), u16x8_min(b.into(), max)).simd_into(self)
}
#[inline(always)]
fn relaxed_narrow_u16x8(self, a: u16x8<Self>, b: u16x8<Self>) -> u8x16<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= u8::MIN as u16 && value <= u8::MAX as u16 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.narrow_u16x8(a, b)
}
#[inline(always)]
fn splat_mask16x8(self, val: bool) -> mask16x8<Self> {
let val: i16 = if val { !0 } else { 0 };
i16x8_splat(val).simd_into(self)
}
#[inline(always)]
fn from_bitmask_mask16x8(self, bits: u64) -> mask16x8<Self> {
let bitset = i16x8_splat(bits as i16);
let powers = u16x8(1, 2, 4, 8, 16, 32, 64, 128);
let selected = v128_and(bitset, powers);
i16x8_ne(selected, i16x8_splat(0)).simd_into(self)
}
#[inline(always)]
fn to_bitmask_mask16x8(self, a: mask16x8<Self>) -> u64 {
i16x8_bitmask(a.into()) as u64
}
#[inline(always)]
fn set_mask16x8(self, a: &mut mask16x8<Self>, index: usize, value: bool) -> () {
assert!(
index < 8usize,
"mask lane index {index} is out of bounds for {} lanes",
8usize
);
let mut lanes: [i16; 8usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn and_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
v128_and(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn or_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
v128_or(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn xor_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
v128_xor(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn not_mask16x8(self, a: mask16x8<Self>) -> mask16x8<Self> {
v128_not(a.into()).simd_into(self)
}
#[inline(always)]
fn select_mask16x8(
self,
a: mask16x8<Self>,
b: mask16x8<Self>,
c: mask16x8<Self>,
) -> mask16x8<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
i16x8_relaxed_laneselect(b.into(), c.into(), a.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
v128_bitselect(b.into(), c.into(), a.into()).simd_into(self)
}
}
#[inline(always)]
fn simd_eq_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x8<Self> {
i16x8_eq(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn any_true_mask16x8(self, a: mask16x8<Self>) -> bool {
v128_any_true(a.into())
}
#[inline(always)]
fn all_true_mask16x8(self, a: mask16x8<Self>) -> bool {
i16x8_all_true(a.into())
}
#[inline(always)]
fn any_false_mask16x8(self, a: mask16x8<Self>) -> bool {
!i16x8_all_true(a.into())
}
#[inline(always)]
fn all_false_mask16x8(self, a: mask16x8<Self>) -> bool {
!v128_any_true(a.into())
}
#[inline(always)]
fn combine_mask16x8(self, a: mask16x8<Self>, b: mask16x8<Self>) -> mask16x16<Self> {
mask16x16 {
val: crate::support::Aligned256([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn splat_i32x4(self, val: i32) -> i32x4<Self> {
i32x4_splat(val).simd_into(self)
}
#[inline(always)]
fn slide_i32x4<const SHIFT: usize>(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = dyn_slide_128(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
i32x4_add(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn sub_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
i32x4_sub(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn mul_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
i32x4_mul(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn and_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
v128_and(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn or_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
v128_or(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn xor_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
v128_xor(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn not_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
v128_not(a.into()).simd_into(self)
}
#[inline(always)]
fn shl_i32x4(self, a: i32x4<Self>, shift: u32) -> i32x4<Self> {
i32x4_shl(a.into(), shift).simd_into(self)
}
#[inline(always)]
fn shlv_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
[
i32::wrapping_shl(a[0usize], b[0usize] as u32),
i32::wrapping_shl(a[1usize], b[1usize] as u32),
i32::wrapping_shl(a[2usize], b[2usize] as u32),
i32::wrapping_shl(a[3usize], b[3usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn shr_i32x4(self, a: i32x4<Self>, shift: u32) -> i32x4<Self> {
i32x4_shr(a.into(), shift).simd_into(self)
}
#[inline(always)]
fn shrv_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
[
i32::wrapping_shr(a[0usize], b[0usize] as u32),
i32::wrapping_shr(a[1usize], b[1usize] as u32),
i32::wrapping_shr(a[2usize], b[2usize] as u32),
i32::wrapping_shr(a[3usize], b[3usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn max_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
i32x4_max(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn min_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
i32x4_min(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_eq_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
i32x4_eq(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_lt_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
i32x4_lt(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_le_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> mask32x4<Self> {
i32x4_le(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn zip_low_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
u32x4_shuffle::<0, 4, 1, 5>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn zip_high_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
u32x4_shuffle::<2, 6, 3, 7>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn unzip_low_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
u32x4_shuffle::<0, 2, 4, 6>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn unzip_high_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x4<Self> {
u32x4_shuffle::<1, 3, 5, 7>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn interleave_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i32x4<Self>, i32x4<Self>) {
(self.zip_low_i32x4(a, b), self.zip_high_i32x4(a, b))
}
#[inline(always)]
fn deinterleave_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> (i32x4<Self>, i32x4<Self>) {
(self.unzip_low_i32x4(a, b), self.unzip_high_i32x4(a, b))
}
#[inline(always)]
fn select_i32x4(self, a: mask32x4<Self>, b: i32x4<Self>, c: i32x4<Self>) -> i32x4<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
i32x4_relaxed_laneselect(b.into(), c.into(), a.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
v128_bitselect(b.into(), c.into(), a.into()).simd_into(self)
}
}
#[inline(always)]
fn combine_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i32x8<Self> {
i32x8 {
val: crate::support::Aligned256([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn neg_i32x4(self, a: i32x4<Self>) -> i32x4<Self> {
i32x4_neg(a.into()).simd_into(self)
}
#[inline(always)]
fn load_four_interleaved_i32x4(self, src: &[i32; 16usize]) -> [i32x4<Self>; 4usize] {
let (chunks, []) = src.as_chunks::<4usize>() else {
unreachable!()
};
let v0: v128 = crate::transmute::checked_transmute_copy::<[i32; 4usize], v128>(&chunks[0]);
let v1: v128 = crate::transmute::checked_transmute_copy::<[i32; 4usize], v128>(&chunks[1]);
let v2: v128 = crate::transmute::checked_transmute_copy::<[i32; 4usize], v128>(&chunks[2]);
let v3: v128 = crate::transmute::checked_transmute_copy::<[i32; 4usize], v128>(&chunks[3]);
let v01_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v1);
let v23_lower = u32x4_shuffle::<0, 4, 1, 5>(v2, v3);
let v01_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v1);
let v23_upper = u32x4_shuffle::<2, 6, 3, 7>(v2, v3);
let out0 = u32x4_shuffle::<0, 1, 4, 5>(v01_lower, v23_lower);
let out1 = u32x4_shuffle::<2, 3, 6, 7>(v01_lower, v23_lower);
let out2 = u32x4_shuffle::<0, 1, 4, 5>(v01_upper, v23_upper);
let out3 = u32x4_shuffle::<2, 3, 6, 7>(v01_upper, v23_upper);
[
out0.simd_into(self),
out1.simd_into(self),
out2.simd_into(self),
out3.simd_into(self),
]
}
#[inline(always)]
fn store_four_interleaved_i32x4(
self,
vectors: [i32x4<Self>; 4usize],
dest: &mut [i32; 16usize],
) -> () {
let v0: v128 = vectors[0].into();
let v1: v128 = vectors[1].into();
let v2: v128 = vectors[2].into();
let v3: v128 = vectors[3].into();
let v02_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v2);
let v13_lower = u32x4_shuffle::<0, 4, 1, 5>(v1, v3);
let v02_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v2);
let v13_upper = u32x4_shuffle::<2, 6, 3, 7>(v1, v3);
let out0 = u32x4_shuffle::<0, 4, 1, 5>(v02_lower, v13_lower);
let out1 = u32x4_shuffle::<2, 6, 3, 7>(v02_lower, v13_lower);
let out2 = u32x4_shuffle::<0, 4, 1, 5>(v02_upper, v13_upper);
let out3 = u32x4_shuffle::<2, 6, 3, 7>(v02_upper, v13_upper);
let (chunks, []) = dest.as_chunks_mut::<4usize>() else {
unreachable!()
};
crate::transmute::checked_transmute_store::<v128, [i32; 4usize]>(out0, &mut chunks[0]);
crate::transmute::checked_transmute_store::<v128, [i32; 4usize]>(out1, &mut chunks[1]);
crate::transmute::checked_transmute_store::<v128, [i32; 4usize]>(out2, &mut chunks[2]);
crate::transmute::checked_transmute_store::<v128, [i32; 4usize]>(out3, &mut chunks[3]);
}
#[inline(always)]
fn widen_i32x4(self, a: i32x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
(
i64x2_extend_low_i32x4(a.into()).simd_into(self),
i64x2_extend_high_i32x4(a.into()).simd_into(self),
)
}
#[inline(always)]
fn narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
let mask = i32x4_splat(65535);
u16x8_narrow_i32x4(v128_and(a.into(), mask), v128_and(b.into(), mask)).simd_into(self)
}
#[inline(always)]
fn saturating_narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
i16x8_narrow_i32x4(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn relaxed_narrow_i32x4(self, a: i32x4<Self>, b: i32x4<Self>) -> i16x8<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= i16::MIN as i32 && value <= i16::MAX as i32 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.saturating_narrow_i32x4(a, b)
}
#[inline(always)]
fn cvt_f32_i32x4(self, a: i32x4<Self>) -> f32x4<Self> {
f32x4_convert_i32x4(a.into()).simd_into(self)
}
#[inline(always)]
fn splat_u32x4(self, val: u32) -> u32x4<Self> {
u32x4_splat(val).simd_into(self)
}
#[inline(always)]
fn slide_u32x4<const SHIFT: usize>(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = dyn_slide_128(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
u32x4_add(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn sub_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
u32x4_sub(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn mul_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
u32x4_mul(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn and_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
v128_and(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn or_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
v128_or(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn xor_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
v128_xor(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn not_u32x4(self, a: u32x4<Self>) -> u32x4<Self> {
v128_not(a.into()).simd_into(self)
}
#[inline(always)]
fn shl_u32x4(self, a: u32x4<Self>, shift: u32) -> u32x4<Self> {
u32x4_shl(a.into(), shift).simd_into(self)
}
#[inline(always)]
fn shlv_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
[
u32::wrapping_shl(a[0usize], b[0usize] as u32),
u32::wrapping_shl(a[1usize], b[1usize] as u32),
u32::wrapping_shl(a[2usize], b[2usize] as u32),
u32::wrapping_shl(a[3usize], b[3usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn shr_u32x4(self, a: u32x4<Self>, shift: u32) -> u32x4<Self> {
u32x4_shr(a.into(), shift).simd_into(self)
}
#[inline(always)]
fn shrv_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
[
u32::wrapping_shr(a[0usize], b[0usize] as u32),
u32::wrapping_shr(a[1usize], b[1usize] as u32),
u32::wrapping_shr(a[2usize], b[2usize] as u32),
u32::wrapping_shr(a[3usize], b[3usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn max_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
u32x4_max(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn min_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
u32x4_min(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_eq_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
u32x4_eq(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_lt_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
u32x4_lt(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_le_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> mask32x4<Self> {
u32x4_le(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn zip_low_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
u32x4_shuffle::<0, 4, 1, 5>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn zip_high_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
u32x4_shuffle::<2, 6, 3, 7>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn unzip_low_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
u32x4_shuffle::<0, 2, 4, 6>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn unzip_high_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x4<Self> {
u32x4_shuffle::<1, 3, 5, 7>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn interleave_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u32x4<Self>, u32x4<Self>) {
(self.zip_low_u32x4(a, b), self.zip_high_u32x4(a, b))
}
#[inline(always)]
fn deinterleave_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> (u32x4<Self>, u32x4<Self>) {
(self.unzip_low_u32x4(a, b), self.unzip_high_u32x4(a, b))
}
#[inline(always)]
fn select_u32x4(self, a: mask32x4<Self>, b: u32x4<Self>, c: u32x4<Self>) -> u32x4<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
i32x4_relaxed_laneselect(b.into(), c.into(), a.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
v128_bitselect(b.into(), c.into(), a.into()).simd_into(self)
}
}
#[inline(always)]
fn combine_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u32x8<Self> {
u32x8 {
val: crate::support::Aligned256([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn load_four_interleaved_u32x4(self, src: &[u32; 16usize]) -> [u32x4<Self>; 4usize] {
let (chunks, []) = src.as_chunks::<4usize>() else {
unreachable!()
};
let v0: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[0]);
let v1: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[1]);
let v2: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[2]);
let v3: v128 = crate::transmute::checked_transmute_copy::<[u32; 4usize], v128>(&chunks[3]);
let v01_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v1);
let v23_lower = u32x4_shuffle::<0, 4, 1, 5>(v2, v3);
let v01_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v1);
let v23_upper = u32x4_shuffle::<2, 6, 3, 7>(v2, v3);
let out0 = u32x4_shuffle::<0, 1, 4, 5>(v01_lower, v23_lower);
let out1 = u32x4_shuffle::<2, 3, 6, 7>(v01_lower, v23_lower);
let out2 = u32x4_shuffle::<0, 1, 4, 5>(v01_upper, v23_upper);
let out3 = u32x4_shuffle::<2, 3, 6, 7>(v01_upper, v23_upper);
[
out0.simd_into(self),
out1.simd_into(self),
out2.simd_into(self),
out3.simd_into(self),
]
}
#[inline(always)]
fn store_four_interleaved_u32x4(
self,
vectors: [u32x4<Self>; 4usize],
dest: &mut [u32; 16usize],
) -> () {
let v0: v128 = vectors[0].into();
let v1: v128 = vectors[1].into();
let v2: v128 = vectors[2].into();
let v3: v128 = vectors[3].into();
let v02_lower = u32x4_shuffle::<0, 4, 1, 5>(v0, v2);
let v13_lower = u32x4_shuffle::<0, 4, 1, 5>(v1, v3);
let v02_upper = u32x4_shuffle::<2, 6, 3, 7>(v0, v2);
let v13_upper = u32x4_shuffle::<2, 6, 3, 7>(v1, v3);
let out0 = u32x4_shuffle::<0, 4, 1, 5>(v02_lower, v13_lower);
let out1 = u32x4_shuffle::<2, 6, 3, 7>(v02_lower, v13_lower);
let out2 = u32x4_shuffle::<0, 4, 1, 5>(v02_upper, v13_upper);
let out3 = u32x4_shuffle::<2, 6, 3, 7>(v02_upper, v13_upper);
let (chunks, []) = dest.as_chunks_mut::<4usize>() else {
unreachable!()
};
crate::transmute::checked_transmute_store::<v128, [u32; 4usize]>(out0, &mut chunks[0]);
crate::transmute::checked_transmute_store::<v128, [u32; 4usize]>(out1, &mut chunks[1]);
crate::transmute::checked_transmute_store::<v128, [u32; 4usize]>(out2, &mut chunks[2]);
crate::transmute::checked_transmute_store::<v128, [u32; 4usize]>(out3, &mut chunks[3]);
}
#[inline(always)]
fn widen_u32x4(self, a: u32x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
(
i64x2_extend_low_u32x4(a.into()).simd_into(self),
i64x2_extend_high_u32x4(a.into()).simd_into(self),
)
}
#[inline(always)]
fn narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
let mask = u32x4_splat(65535);
u16x8_narrow_i32x4(v128_and(a.into(), mask), v128_and(b.into(), mask)).simd_into(self)
}
#[inline(always)]
fn saturating_narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
let max = u32x4_splat(65535);
u16x8_narrow_i32x4(u32x4_min(a.into(), max), u32x4_min(b.into(), max)).simd_into(self)
}
#[inline(always)]
fn relaxed_narrow_u32x4(self, a: u32x4<Self>, b: u32x4<Self>) -> u16x8<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= u16::MIN as u32 && value <= u16::MAX as u32 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.narrow_u32x4(a, b)
}
#[inline(always)]
fn cvt_f32_u32x4(self, a: u32x4<Self>) -> f32x4<Self> {
f32x4_convert_u32x4(a.into()).simd_into(self)
}
#[inline(always)]
fn splat_mask32x4(self, val: bool) -> mask32x4<Self> {
let val: i32 = if val { !0 } else { 0 };
i32x4_splat(val).simd_into(self)
}
#[inline(always)]
fn from_bitmask_mask32x4(self, bits: u64) -> mask32x4<Self> {
let bitset = i32x4_splat(bits as i32);
let powers = u32x4(1, 2, 4, 8);
let selected = v128_and(bitset, powers);
i32x4_ne(selected, i32x4_splat(0)).simd_into(self)
}
#[inline(always)]
fn to_bitmask_mask32x4(self, a: mask32x4<Self>) -> u64 {
i32x4_bitmask(a.into()) as u64
}
#[inline(always)]
fn set_mask32x4(self, a: &mut mask32x4<Self>, index: usize, value: bool) -> () {
assert!(
index < 4usize,
"mask lane index {index} is out of bounds for {} lanes",
4usize
);
let mut lanes: [i32; 4usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn and_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
v128_and(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn or_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
v128_or(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn xor_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
v128_xor(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn not_mask32x4(self, a: mask32x4<Self>) -> mask32x4<Self> {
v128_not(a.into()).simd_into(self)
}
#[inline(always)]
fn select_mask32x4(
self,
a: mask32x4<Self>,
b: mask32x4<Self>,
c: mask32x4<Self>,
) -> mask32x4<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
i32x4_relaxed_laneselect(b.into(), c.into(), a.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
v128_bitselect(b.into(), c.into(), a.into()).simd_into(self)
}
}
#[inline(always)]
fn simd_eq_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x4<Self> {
i32x4_eq(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn any_true_mask32x4(self, a: mask32x4<Self>) -> bool {
v128_any_true(a.into())
}
#[inline(always)]
fn all_true_mask32x4(self, a: mask32x4<Self>) -> bool {
i32x4_all_true(a.into())
}
#[inline(always)]
fn any_false_mask32x4(self, a: mask32x4<Self>) -> bool {
!i32x4_all_true(a.into())
}
#[inline(always)]
fn all_false_mask32x4(self, a: mask32x4<Self>) -> bool {
!v128_any_true(a.into())
}
#[inline(always)]
fn combine_mask32x4(self, a: mask32x4<Self>, b: mask32x4<Self>) -> mask32x8<Self> {
mask32x8 {
val: crate::support::Aligned256([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn splat_f64x2(self, val: f64) -> f64x2<Self> {
f64x2_splat(val).simd_into(self)
}
#[inline(always)]
fn slide_f64x2<const SHIFT: usize>(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
if SHIFT >= 2usize {
return b;
}
let result = dyn_slide_128(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn abs_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
f64x2_abs(a.into()).simd_into(self)
}
#[inline(always)]
fn neg_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
f64x2_neg(a.into()).simd_into(self)
}
#[inline(always)]
fn sqrt_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
f64x2_sqrt(a.into()).simd_into(self)
}
#[inline(always)]
fn approximate_recip_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
1.0 / a
}
#[inline(always)]
fn add_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
f64x2_add(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn sub_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
f64x2_sub(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn mul_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
f64x2_mul(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn div_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
f64x2_div(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn copysign_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
let sign_mask = f64x2_splat(-0.0_f64);
let sign_bits = v128_and(b.into(), sign_mask.into());
let magnitude = v128_andnot(a.into(), sign_mask.into());
v128_or(magnitude, sign_bits).simd_into(self)
}
#[inline(always)]
fn max_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
f64x2_relaxed_max(a.into(), b.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
f64x2_max(a.into(), b.into()).simd_into(self)
}
}
#[inline(always)]
fn min_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
f64x2_relaxed_min(a.into(), b.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
f64x2_min(a.into(), b.into()).simd_into(self)
}
}
#[inline(always)]
fn max_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
let intermediate = f64x2_pmax(b.into(), a.into());
let b_is_nan = f64x2_ne(b.into(), b.into());
v128_bitselect(a.into(), intermediate, b_is_nan).simd_into(self)
}
#[inline(always)]
fn min_precise_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
let intermediate = f64x2_pmin(b.into(), a.into());
let b_is_nan = f64x2_ne(b.into(), b.into());
v128_bitselect(a.into(), intermediate, b_is_nan).simd_into(self)
}
#[inline(always)]
fn simd_eq_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
f64x2_eq(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_lt_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
f64x2_lt(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_le_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> mask64x2<Self> {
f64x2_le(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn zip_low_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
u64x2_shuffle::<0, 2>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn zip_high_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
u64x2_shuffle::<1, 3>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn unzip_low_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
u64x2_shuffle::<0, 2>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn unzip_high_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x2<Self> {
u64x2_shuffle::<1, 3>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn interleave_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> (f64x2<Self>, f64x2<Self>) {
(self.zip_low_f64x2(a, b), self.zip_high_f64x2(a, b))
}
#[inline(always)]
fn deinterleave_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> (f64x2<Self>, f64x2<Self>) {
(self.unzip_low_f64x2(a, b), self.unzip_high_f64x2(a, b))
}
#[inline(always)]
fn mul_add_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
f64x2_relaxed_madd(a.into(), b.into(), c.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
self.add_f64x2(self.mul_f64x2(a, b), c)
}
}
#[inline(always)]
fn mul_sub_f64x2(self, a: f64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
f64x2_relaxed_madd(a.into(), b.into(), f64x2_neg(c.into())).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
self.sub_f64x2(self.mul_f64x2(a, b), c)
}
}
#[inline(always)]
fn floor_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
f64x2_floor(a.into()).simd_into(self)
}
#[inline(always)]
fn ceil_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
f64x2_ceil(a.into()).simd_into(self)
}
#[inline(always)]
fn round_ties_even_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
f64x2_nearest(a.into()).simd_into(self)
}
#[inline(always)]
fn fract_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
self.sub_f64x2(a, self.trunc_f64x2(a))
}
#[inline(always)]
fn trunc_f64x2(self, a: f64x2<Self>) -> f64x2<Self> {
f64x2_trunc(a.into()).simd_into(self)
}
#[inline(always)]
fn select_f64x2(self, a: mask64x2<Self>, b: f64x2<Self>, c: f64x2<Self>) -> f64x2<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
i64x2_relaxed_laneselect(b.into(), c.into(), a.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
v128_bitselect(b.into(), c.into(), a.into()).simd_into(self)
}
}
#[inline(always)]
fn combine_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f64x4<Self> {
f64x4 {
val: crate::support::Aligned256([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn load_four_interleaved_f64x2(self, src: &[f64; 8usize]) -> [f64x2<Self>; 4usize] {
let (chunks, []) = src.as_chunks::<2usize>() else {
unreachable!()
};
let v0: v128 = crate::transmute::checked_transmute_copy::<[f64; 2usize], v128>(&chunks[0]);
let v1: v128 = crate::transmute::checked_transmute_copy::<[f64; 2usize], v128>(&chunks[1]);
let v2: v128 = crate::transmute::checked_transmute_copy::<[f64; 2usize], v128>(&chunks[2]);
let v3: v128 = crate::transmute::checked_transmute_copy::<[f64; 2usize], v128>(&chunks[3]);
let out0 = u64x2_shuffle::<0, 2>(v0, v2);
let out1 = u64x2_shuffle::<1, 3>(v0, v2);
let out2 = u64x2_shuffle::<0, 2>(v1, v3);
let out3 = u64x2_shuffle::<1, 3>(v1, v3);
[
out0.simd_into(self),
out1.simd_into(self),
out2.simd_into(self),
out3.simd_into(self),
]
}
#[inline(always)]
fn store_four_interleaved_f64x2(
self,
vectors: [f64x2<Self>; 4usize],
dest: &mut [f64; 8usize],
) -> () {
let v0: v128 = vectors[0].into();
let v1: v128 = vectors[1].into();
let v2: v128 = vectors[2].into();
let v3: v128 = vectors[3].into();
let out0 = u64x2_shuffle::<0, 2>(v0, v1);
let out1 = u64x2_shuffle::<0, 2>(v2, v3);
let out2 = u64x2_shuffle::<1, 3>(v0, v1);
let out3 = u64x2_shuffle::<1, 3>(v2, v3);
let (chunks, []) = dest.as_chunks_mut::<2usize>() else {
unreachable!()
};
crate::transmute::checked_transmute_store::<v128, [f64; 2usize]>(out0, &mut chunks[0]);
crate::transmute::checked_transmute_store::<v128, [f64; 2usize]>(out1, &mut chunks[1]);
crate::transmute::checked_transmute_store::<v128, [f64; 2usize]>(out2, &mut chunks[2]);
crate::transmute::checked_transmute_store::<v128, [f64; 2usize]>(out3, &mut chunks[3]);
}
#[inline(always)]
fn narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
let low = f32x4_demote_f64x2_zero(a.into());
let high = f32x4_demote_f64x2_zero(b.into());
i64x2_shuffle::<0, 2>(low, high).simd_into(self)
}
#[inline(always)]
fn saturating_narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
self.narrow_f64x2(a, b)
}
#[inline(always)]
fn relaxed_narrow_f64x2(self, a: f64x2<Self>, b: f64x2<Self>) -> f32x4<Self> {
self.narrow_f64x2(a, b)
}
#[inline(always)]
fn cvt_u64_f64x2(self, a: f64x2<Self>) -> u64x2<Self> {
[a[0usize] as u64, a[1usize] as u64].simd_into(self)
}
#[inline(always)]
fn cvt_u64_precise_f64x2(self, a: f64x2<Self>) -> u64x2<Self> {
[a[0usize] as u64, a[1usize] as u64].simd_into(self)
}
#[inline(always)]
fn cvt_i64_f64x2(self, a: f64x2<Self>) -> i64x2<Self> {
[a[0usize] as i64, a[1usize] as i64].simd_into(self)
}
#[inline(always)]
fn cvt_i64_precise_f64x2(self, a: f64x2<Self>) -> i64x2<Self> {
[a[0usize] as i64, a[1usize] as i64].simd_into(self)
}
#[inline(always)]
fn splat_i64x2(self, val: i64) -> i64x2<Self> {
i64x2_splat(val).simd_into(self)
}
#[inline(always)]
fn slide_i64x2<const SHIFT: usize>(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
if SHIFT >= 2usize {
return b;
}
let result = dyn_slide_128(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
i64x2_add(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn sub_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
i64x2_sub(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn mul_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
i64x2_mul(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn and_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
v128_and(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn or_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
v128_or(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn xor_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
v128_xor(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn not_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
v128_not(a.into()).simd_into(self)
}
#[inline(always)]
fn shl_i64x2(self, a: i64x2<Self>, shift: u32) -> i64x2<Self> {
i64x2_shl(a.into(), shift).simd_into(self)
}
#[inline(always)]
fn shlv_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
[
i64::wrapping_shl(a[0usize], b[0usize] as u32),
i64::wrapping_shl(a[1usize], b[1usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn shr_i64x2(self, a: i64x2<Self>, shift: u32) -> i64x2<Self> {
i64x2_shr(a.into(), shift).simd_into(self)
}
#[inline(always)]
fn shrv_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
[
i64::wrapping_shr(a[0usize], b[0usize] as u32),
i64::wrapping_shr(a[1usize], b[1usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn max_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
[
i64::max(a[0usize], b[0usize]),
i64::max(a[1usize], b[1usize]),
]
.simd_into(self)
}
#[inline(always)]
fn min_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
[
i64::min(a[0usize], b[0usize]),
i64::min(a[1usize], b[1usize]),
]
.simd_into(self)
}
#[inline(always)]
fn simd_eq_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
i64x2_eq(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_lt_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
i64x2_lt(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn simd_le_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> mask64x2<Self> {
i64x2_le(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn zip_low_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
u64x2_shuffle::<0, 2>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn zip_high_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
u64x2_shuffle::<1, 3>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn unzip_low_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
u64x2_shuffle::<0, 2>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn unzip_high_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x2<Self> {
u64x2_shuffle::<1, 3>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn interleave_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> (i64x2<Self>, i64x2<Self>) {
(self.zip_low_i64x2(a, b), self.zip_high_i64x2(a, b))
}
#[inline(always)]
fn deinterleave_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> (i64x2<Self>, i64x2<Self>) {
(self.unzip_low_i64x2(a, b), self.unzip_high_i64x2(a, b))
}
#[inline(always)]
fn select_i64x2(self, a: mask64x2<Self>, b: i64x2<Self>, c: i64x2<Self>) -> i64x2<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
i64x2_relaxed_laneselect(b.into(), c.into(), a.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
v128_bitselect(b.into(), c.into(), a.into()).simd_into(self)
}
}
#[inline(always)]
fn combine_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i64x4<Self> {
i64x4 {
val: crate::support::Aligned256([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn neg_i64x2(self, a: i64x2<Self>) -> i64x2<Self> {
i64x2_neg(a.into()).simd_into(self)
}
#[inline(always)]
fn load_four_interleaved_i64x2(self, src: &[i64; 8usize]) -> [i64x2<Self>; 4usize] {
let (chunks, []) = src.as_chunks::<2usize>() else {
unreachable!()
};
let v0: v128 = crate::transmute::checked_transmute_copy::<[i64; 2usize], v128>(&chunks[0]);
let v1: v128 = crate::transmute::checked_transmute_copy::<[i64; 2usize], v128>(&chunks[1]);
let v2: v128 = crate::transmute::checked_transmute_copy::<[i64; 2usize], v128>(&chunks[2]);
let v3: v128 = crate::transmute::checked_transmute_copy::<[i64; 2usize], v128>(&chunks[3]);
let out0 = u64x2_shuffle::<0, 2>(v0, v2);
let out1 = u64x2_shuffle::<1, 3>(v0, v2);
let out2 = u64x2_shuffle::<0, 2>(v1, v3);
let out3 = u64x2_shuffle::<1, 3>(v1, v3);
[
out0.simd_into(self),
out1.simd_into(self),
out2.simd_into(self),
out3.simd_into(self),
]
}
#[inline(always)]
fn store_four_interleaved_i64x2(
self,
vectors: [i64x2<Self>; 4usize],
dest: &mut [i64; 8usize],
) -> () {
let v0: v128 = vectors[0].into();
let v1: v128 = vectors[1].into();
let v2: v128 = vectors[2].into();
let v3: v128 = vectors[3].into();
let out0 = u64x2_shuffle::<0, 2>(v0, v1);
let out1 = u64x2_shuffle::<0, 2>(v2, v3);
let out2 = u64x2_shuffle::<1, 3>(v0, v1);
let out3 = u64x2_shuffle::<1, 3>(v2, v3);
let (chunks, []) = dest.as_chunks_mut::<2usize>() else {
unreachable!()
};
crate::transmute::checked_transmute_store::<v128, [i64; 2usize]>(out0, &mut chunks[0]);
crate::transmute::checked_transmute_store::<v128, [i64; 2usize]>(out1, &mut chunks[1]);
crate::transmute::checked_transmute_store::<v128, [i64; 2usize]>(out2, &mut chunks[2]);
crate::transmute::checked_transmute_store::<v128, [i64; 2usize]>(out3, &mut chunks[3]);
}
#[inline(always)]
fn narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
i8x16_shuffle::<0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27>(
a.into(),
b.into(),
)
.simd_into(self)
}
#[inline(always)]
fn saturating_narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
[
a[0usize].clamp(i32::MIN as i64, i32::MAX as i64) as i32,
a[1usize].clamp(i32::MIN as i64, i32::MAX as i64) as i32,
b[0usize].clamp(i32::MIN as i64, i32::MAX as i64) as i32,
b[1usize].clamp(i32::MIN as i64, i32::MAX as i64) as i32,
]
.simd_into(self)
}
#[inline(always)]
fn relaxed_narrow_i64x2(self, a: i64x2<Self>, b: i64x2<Self>) -> i32x4<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= i32::MIN as i64 && value <= i32::MAX as i64 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.narrow_i64x2(a, b)
}
#[inline(always)]
fn cvt_f64_i64x2(self, a: i64x2<Self>) -> f64x2<Self> {
[a[0usize] as f64, a[1usize] as f64].simd_into(self)
}
#[inline(always)]
fn splat_u64x2(self, val: u64) -> u64x2<Self> {
u64x2_splat(val).simd_into(self)
}
#[inline(always)]
fn slide_u64x2<const SHIFT: usize>(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
if SHIFT >= 2usize {
return b;
}
let result = dyn_slide_128(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x16 {
val: crate::support::Aligned128(result),
simd: self,
})
}
#[inline(always)]
fn add_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
u64x2_add(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn sub_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
u64x2_sub(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn mul_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
u64x2_mul(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn and_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
v128_and(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn or_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
v128_or(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn xor_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
v128_xor(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn not_u64x2(self, a: u64x2<Self>) -> u64x2<Self> {
v128_not(a.into()).simd_into(self)
}
#[inline(always)]
fn shl_u64x2(self, a: u64x2<Self>, shift: u32) -> u64x2<Self> {
u64x2_shl(a.into(), shift).simd_into(self)
}
#[inline(always)]
fn shlv_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
[
u64::wrapping_shl(a[0usize], b[0usize] as u32),
u64::wrapping_shl(a[1usize], b[1usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn shr_u64x2(self, a: u64x2<Self>, shift: u32) -> u64x2<Self> {
u64x2_shr(a.into(), shift).simd_into(self)
}
#[inline(always)]
fn shrv_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
[
u64::wrapping_shr(a[0usize], b[0usize] as u32),
u64::wrapping_shr(a[1usize], b[1usize] as u32),
]
.simd_into(self)
}
#[inline(always)]
fn max_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
[
u64::max(a[0usize], b[0usize]),
u64::max(a[1usize], b[1usize]),
]
.simd_into(self)
}
#[inline(always)]
fn min_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
[
u64::min(a[0usize], b[0usize]),
u64::min(a[1usize], b[1usize]),
]
.simd_into(self)
}
#[inline(always)]
fn simd_eq_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
[
-(u64::eq(&a[0usize], &b[0usize]) as i64),
-(u64::eq(&a[1usize], &b[1usize]) as i64),
]
.simd_into(self)
}
#[inline(always)]
fn simd_lt_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
[
-(u64::lt(&a[0usize], &b[0usize]) as i64),
-(u64::lt(&a[1usize], &b[1usize]) as i64),
]
.simd_into(self)
}
#[inline(always)]
fn simd_le_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> mask64x2<Self> {
[
-(u64::le(&a[0usize], &b[0usize]) as i64),
-(u64::le(&a[1usize], &b[1usize]) as i64),
]
.simd_into(self)
}
#[inline(always)]
fn zip_low_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
u64x2_shuffle::<0, 2>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn zip_high_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
u64x2_shuffle::<1, 3>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn unzip_low_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
u64x2_shuffle::<0, 2>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn unzip_high_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x2<Self> {
u64x2_shuffle::<1, 3>(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn interleave_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> (u64x2<Self>, u64x2<Self>) {
(self.zip_low_u64x2(a, b), self.zip_high_u64x2(a, b))
}
#[inline(always)]
fn deinterleave_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> (u64x2<Self>, u64x2<Self>) {
(self.unzip_low_u64x2(a, b), self.unzip_high_u64x2(a, b))
}
#[inline(always)]
fn select_u64x2(self, a: mask64x2<Self>, b: u64x2<Self>, c: u64x2<Self>) -> u64x2<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
i64x2_relaxed_laneselect(b.into(), c.into(), a.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
v128_bitselect(b.into(), c.into(), a.into()).simd_into(self)
}
}
#[inline(always)]
fn combine_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u64x4<Self> {
u64x4 {
val: crate::support::Aligned256([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn load_four_interleaved_u64x2(self, src: &[u64; 8usize]) -> [u64x2<Self>; 4usize] {
let (chunks, []) = src.as_chunks::<2usize>() else {
unreachable!()
};
let v0: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[0]);
let v1: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[1]);
let v2: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[2]);
let v3: v128 = crate::transmute::checked_transmute_copy::<[u64; 2usize], v128>(&chunks[3]);
let out0 = u64x2_shuffle::<0, 2>(v0, v2);
let out1 = u64x2_shuffle::<1, 3>(v0, v2);
let out2 = u64x2_shuffle::<0, 2>(v1, v3);
let out3 = u64x2_shuffle::<1, 3>(v1, v3);
[
out0.simd_into(self),
out1.simd_into(self),
out2.simd_into(self),
out3.simd_into(self),
]
}
#[inline(always)]
fn store_four_interleaved_u64x2(
self,
vectors: [u64x2<Self>; 4usize],
dest: &mut [u64; 8usize],
) -> () {
let v0: v128 = vectors[0].into();
let v1: v128 = vectors[1].into();
let v2: v128 = vectors[2].into();
let v3: v128 = vectors[3].into();
let out0 = u64x2_shuffle::<0, 2>(v0, v1);
let out1 = u64x2_shuffle::<0, 2>(v2, v3);
let out2 = u64x2_shuffle::<1, 3>(v0, v1);
let out3 = u64x2_shuffle::<1, 3>(v2, v3);
let (chunks, []) = dest.as_chunks_mut::<2usize>() else {
unreachable!()
};
crate::transmute::checked_transmute_store::<v128, [u64; 2usize]>(out0, &mut chunks[0]);
crate::transmute::checked_transmute_store::<v128, [u64; 2usize]>(out1, &mut chunks[1]);
crate::transmute::checked_transmute_store::<v128, [u64; 2usize]>(out2, &mut chunks[2]);
crate::transmute::checked_transmute_store::<v128, [u64; 2usize]>(out3, &mut chunks[3]);
}
#[inline(always)]
fn narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
i8x16_shuffle::<0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27>(
a.into(),
b.into(),
)
.simd_into(self)
}
#[inline(always)]
fn saturating_narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
[
a[0usize].clamp(u32::MIN as u64, u32::MAX as u64) as u32,
a[1usize].clamp(u32::MIN as u64, u32::MAX as u64) as u32,
b[0usize].clamp(u32::MIN as u64, u32::MAX as u64) as u32,
b[1usize].clamp(u32::MIN as u64, u32::MAX as u64) as u32,
]
.simd_into(self)
}
#[inline(always)]
fn relaxed_narrow_u64x2(self, a: u64x2<Self>, b: u64x2<Self>) -> u32x4<Self> {
debug_assert!(
a.as_slice()
.iter()
.chain(b.as_slice())
.all(|&value| { value >= u32::MIN as u64 && value <= u32::MAX as u64 }),
"relaxed_narrow inputs must fit in the destination type",
);
self.narrow_u64x2(a, b)
}
#[inline(always)]
fn cvt_f64_u64x2(self, a: u64x2<Self>) -> f64x2<Self> {
[a[0usize] as f64, a[1usize] as f64].simd_into(self)
}
#[inline(always)]
fn splat_mask64x2(self, val: bool) -> mask64x2<Self> {
let val: i64 = if val { !0 } else { 0 };
i64x2_splat(val).simd_into(self)
}
#[inline(always)]
fn from_bitmask_mask64x2(self, bits: u64) -> mask64x2<Self> {
let bitset = i64x2_splat(bits as i64);
let powers = u64x2(1, 2);
let selected = v128_and(bitset, powers);
i64x2_ne(selected, i64x2_splat(0)).simd_into(self)
}
#[inline(always)]
fn to_bitmask_mask64x2(self, a: mask64x2<Self>) -> u64 {
i64x2_bitmask(a.into()) as u64
}
#[inline(always)]
fn set_mask64x2(self, a: &mut mask64x2<Self>, index: usize, value: bool) -> () {
assert!(
index < 2usize,
"mask lane index {index} is out of bounds for {} lanes",
2usize
);
let mut lanes: [i64; 2usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn and_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
v128_and(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn or_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
v128_or(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn xor_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
v128_xor(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn not_mask64x2(self, a: mask64x2<Self>) -> mask64x2<Self> {
v128_not(a.into()).simd_into(self)
}
#[inline(always)]
fn select_mask64x2(
self,
a: mask64x2<Self>,
b: mask64x2<Self>,
c: mask64x2<Self>,
) -> mask64x2<Self> {
#[cfg(target_feature = "relaxed-simd")]
{
i64x2_relaxed_laneselect(b.into(), c.into(), a.into()).simd_into(self)
}
#[cfg(not(target_feature = "relaxed-simd"))]
{
v128_bitselect(b.into(), c.into(), a.into()).simd_into(self)
}
}
#[inline(always)]
fn simd_eq_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x2<Self> {
i64x2_eq(a.into(), b.into()).simd_into(self)
}
#[inline(always)]
fn any_true_mask64x2(self, a: mask64x2<Self>) -> bool {
v128_any_true(a.into())
}
#[inline(always)]
fn all_true_mask64x2(self, a: mask64x2<Self>) -> bool {
i64x2_all_true(a.into())
}
#[inline(always)]
fn any_false_mask64x2(self, a: mask64x2<Self>) -> bool {
!i64x2_all_true(a.into())
}
#[inline(always)]
fn all_false_mask64x2(self, a: mask64x2<Self>) -> bool {
!v128_any_true(a.into())
}
#[inline(always)]
fn combine_mask64x2(self, a: mask64x2<Self>, b: mask64x2<Self>) -> mask64x4<Self> {
mask64x4 {
val: crate::support::Aligned256([a.val.0, b.val.0]),
simd: self,
}
}
#[inline(always)]
fn slide_f32x8<const SHIFT: usize>(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = cross_block_slide_128x2(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn combine_f32x8(self, a: f32x8<Self>, b: f32x8<Self>) -> f32x16<Self> {
f32x16 {
val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
simd: self,
}
}
#[inline(always)]
fn split_f32x8(self, a: f32x8<Self>) -> (f32x4<Self>, f32x4<Self>) {
(
f32x4 {
val: crate::support::Aligned128(a.val.0[0]),
simd: self,
},
f32x4 {
val: crate::support::Aligned128(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn slide_i8x32<const SHIFT: usize>(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x32<Self> {
if SHIFT >= 32usize {
return b;
}
let result =
cross_block_slide_128x2(Bytes::to_bytes(a).val.0, Bytes::to_bytes(b).val.0, SHIFT);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn combine_i8x32(self, a: i8x32<Self>, b: i8x32<Self>) -> i8x64<Self> {
i8x64 {
val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
simd: self,
}
}
#[inline(always)]
fn split_i8x32(self, a: i8x32<Self>) -> (i8x16<Self>, i8x16<Self>) {
(
i8x16 {
val: crate::support::Aligned128(a.val.0[0]),
simd: self,
},
i8x16 {
val: crate::support::Aligned128(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn slide_u8x32<const SHIFT: usize>(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x32<Self> {
if SHIFT >= 32usize {
return b;
}
let result =
cross_block_slide_128x2(Bytes::to_bytes(a).val.0, Bytes::to_bytes(b).val.0, SHIFT);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
self.swizzle_dyn_precise_u8x32(a, indices)
}
#[inline(always)]
fn swizzle_dyn_precise_u8x32(self, a: u8x32<Self>, indices: u8x32<Self>) -> u8x32<Self> {
let bytes = Bytes::to_bytes(a);
let (table_low, table_high) = self.split_u8x32(bytes);
let (indices_low, indices_high) = self.split_u8x32(indices);
let high_table_offset = self.splat_u8x16(16);
let output_low_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_low);
let output_low_from_high = self
.swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_low, high_table_offset));
let output_low = self.or_u8x16(output_low_from_low, output_low_from_high);
let output_high_from_low = self.swizzle_dyn_precise_u8x16(table_low, indices_high);
let output_high_from_high = self
.swizzle_dyn_precise_u8x16(table_high, self.sub_u8x16(indices_high, high_table_offset));
let output_high = self.or_u8x16(output_high_from_low, output_high_from_high);
let result_bytes = self.combine_u8x16(output_low, output_high);
Bytes::from_bytes(result_bytes)
}
#[inline(always)]
fn combine_u8x32(self, a: u8x32<Self>, b: u8x32<Self>) -> u8x64<Self> {
u8x64 {
val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
simd: self,
}
}
#[inline(always)]
fn split_u8x32(self, a: u8x32<Self>) -> (u8x16<Self>, u8x16<Self>) {
(
u8x16 {
val: crate::support::Aligned128(a.val.0[0]),
simd: self,
},
u8x16 {
val: crate::support::Aligned128(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn set_mask8x32(self, a: &mut mask8x32<Self>, index: usize, value: bool) -> () {
assert!(
index < 32usize,
"mask lane index {index} is out of bounds for {} lanes",
32usize
);
let mut lanes: [i8; 32usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn combine_mask8x32(self, a: mask8x32<Self>, b: mask8x32<Self>) -> mask8x64<Self> {
mask8x64 {
val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
simd: self,
}
}
#[inline(always)]
fn split_mask8x32(self, a: mask8x32<Self>) -> (mask8x16<Self>, mask8x16<Self>) {
(
mask8x16 {
val: crate::support::Aligned128(a.val.0[0]),
simd: self,
},
mask8x16 {
val: crate::support::Aligned128(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn slide_i16x16<const SHIFT: usize>(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = cross_block_slide_128x2(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 2usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn combine_i16x16(self, a: i16x16<Self>, b: i16x16<Self>) -> i16x32<Self> {
i16x32 {
val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
simd: self,
}
}
#[inline(always)]
fn split_i16x16(self, a: i16x16<Self>) -> (i16x8<Self>, i16x8<Self>) {
(
i16x8 {
val: crate::support::Aligned128(a.val.0[0]),
simd: self,
},
i16x8 {
val: crate::support::Aligned128(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn slide_u16x16<const SHIFT: usize>(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = cross_block_slide_128x2(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 2usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn combine_u16x16(self, a: u16x16<Self>, b: u16x16<Self>) -> u16x32<Self> {
u16x32 {
val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
simd: self,
}
}
#[inline(always)]
fn split_u16x16(self, a: u16x16<Self>) -> (u16x8<Self>, u16x8<Self>) {
(
u16x8 {
val: crate::support::Aligned128(a.val.0[0]),
simd: self,
},
u16x8 {
val: crate::support::Aligned128(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn set_mask16x16(self, a: &mut mask16x16<Self>, index: usize, value: bool) -> () {
assert!(
index < 16usize,
"mask lane index {index} is out of bounds for {} lanes",
16usize
);
let mut lanes: [i16; 16usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn combine_mask16x16(self, a: mask16x16<Self>, b: mask16x16<Self>) -> mask16x32<Self> {
mask16x32 {
val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
simd: self,
}
}
#[inline(always)]
fn split_mask16x16(self, a: mask16x16<Self>) -> (mask16x8<Self>, mask16x8<Self>) {
(
mask16x8 {
val: crate::support::Aligned128(a.val.0[0]),
simd: self,
},
mask16x8 {
val: crate::support::Aligned128(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn slide_i32x8<const SHIFT: usize>(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = cross_block_slide_128x2(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn combine_i32x8(self, a: i32x8<Self>, b: i32x8<Self>) -> i32x16<Self> {
i32x16 {
val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
simd: self,
}
}
#[inline(always)]
fn split_i32x8(self, a: i32x8<Self>) -> (i32x4<Self>, i32x4<Self>) {
(
i32x4 {
val: crate::support::Aligned128(a.val.0[0]),
simd: self,
},
i32x4 {
val: crate::support::Aligned128(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn slide_u32x8<const SHIFT: usize>(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = cross_block_slide_128x2(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn combine_u32x8(self, a: u32x8<Self>, b: u32x8<Self>) -> u32x16<Self> {
u32x16 {
val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
simd: self,
}
}
#[inline(always)]
fn split_u32x8(self, a: u32x8<Self>) -> (u32x4<Self>, u32x4<Self>) {
(
u32x4 {
val: crate::support::Aligned128(a.val.0[0]),
simd: self,
},
u32x4 {
val: crate::support::Aligned128(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn set_mask32x8(self, a: &mut mask32x8<Self>, index: usize, value: bool) -> () {
assert!(
index < 8usize,
"mask lane index {index} is out of bounds for {} lanes",
8usize
);
let mut lanes: [i32; 8usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn combine_mask32x8(self, a: mask32x8<Self>, b: mask32x8<Self>) -> mask32x16<Self> {
mask32x16 {
val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
simd: self,
}
}
#[inline(always)]
fn split_mask32x8(self, a: mask32x8<Self>) -> (mask32x4<Self>, mask32x4<Self>) {
(
mask32x4 {
val: crate::support::Aligned128(a.val.0[0]),
simd: self,
},
mask32x4 {
val: crate::support::Aligned128(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn slide_f64x4<const SHIFT: usize>(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = cross_block_slide_128x2(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn combine_f64x4(self, a: f64x4<Self>, b: f64x4<Self>) -> f64x8<Self> {
f64x8 {
val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
simd: self,
}
}
#[inline(always)]
fn split_f64x4(self, a: f64x4<Self>) -> (f64x2<Self>, f64x2<Self>) {
(
f64x2 {
val: crate::support::Aligned128(a.val.0[0]),
simd: self,
},
f64x2 {
val: crate::support::Aligned128(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn slide_i64x4<const SHIFT: usize>(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = cross_block_slide_128x2(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn combine_i64x4(self, a: i64x4<Self>, b: i64x4<Self>) -> i64x8<Self> {
i64x8 {
val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
simd: self,
}
}
#[inline(always)]
fn split_i64x4(self, a: i64x4<Self>) -> (i64x2<Self>, i64x2<Self>) {
(
i64x2 {
val: crate::support::Aligned128(a.val.0[0]),
simd: self,
},
i64x2 {
val: crate::support::Aligned128(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn slide_u64x4<const SHIFT: usize>(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x4<Self> {
if SHIFT >= 4usize {
return b;
}
let result = cross_block_slide_128x2(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x32 {
val: crate::support::Aligned256(result),
simd: self,
})
}
#[inline(always)]
fn combine_u64x4(self, a: u64x4<Self>, b: u64x4<Self>) -> u64x8<Self> {
u64x8 {
val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
simd: self,
}
}
#[inline(always)]
fn split_u64x4(self, a: u64x4<Self>) -> (u64x2<Self>, u64x2<Self>) {
(
u64x2 {
val: crate::support::Aligned128(a.val.0[0]),
simd: self,
},
u64x2 {
val: crate::support::Aligned128(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn set_mask64x4(self, a: &mut mask64x4<Self>, index: usize, value: bool) -> () {
assert!(
index < 4usize,
"mask lane index {index} is out of bounds for {} lanes",
4usize
);
let mut lanes: [i64; 4usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn combine_mask64x4(self, a: mask64x4<Self>, b: mask64x4<Self>) -> mask64x8<Self> {
mask64x8 {
val: crate::support::Aligned512([a.val.0[0], a.val.0[1], b.val.0[0], b.val.0[1]]),
simd: self,
}
}
#[inline(always)]
fn split_mask64x4(self, a: mask64x4<Self>) -> (mask64x2<Self>, mask64x2<Self>) {
(
mask64x2 {
val: crate::support::Aligned128(a.val.0[0]),
simd: self,
},
mask64x2 {
val: crate::support::Aligned128(a.val.0[1]),
simd: self,
},
)
}
#[inline(always)]
fn slide_f32x16<const SHIFT: usize>(self, a: f32x16<Self>, b: f32x16<Self>) -> f32x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = cross_block_slide_128x4(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_f32x16(self, a: f32x16<Self>) -> (f32x8<Self>, f32x8<Self>) {
(
f32x8 {
val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
simd: self,
},
f32x8 {
val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
simd: self,
},
)
}
#[inline(always)]
fn slide_i8x64<const SHIFT: usize>(self, a: i8x64<Self>, b: i8x64<Self>) -> i8x64<Self> {
if SHIFT >= 64usize {
return b;
}
let result =
cross_block_slide_128x4(Bytes::to_bytes(a).val.0, Bytes::to_bytes(b).val.0, SHIFT);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_i8x64(self, a: i8x64<Self>) -> (i8x32<Self>, i8x32<Self>) {
(
i8x32 {
val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
simd: self,
},
i8x32 {
val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
simd: self,
},
)
}
#[inline(always)]
fn slide_u8x64<const SHIFT: usize>(self, a: u8x64<Self>, b: u8x64<Self>) -> u8x64<Self> {
if SHIFT >= 64usize {
return b;
}
let result =
cross_block_slide_128x4(Bytes::to_bytes(a).val.0, Bytes::to_bytes(b).val.0, SHIFT);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn swizzle_dyn_u8x64(self, a: u8x64<Self>, indices: u8x64<Self>) -> u8x64<Self> {
self.swizzle_dyn_precise_u8x64(a, indices)
}
#[inline(always)]
fn swizzle_dyn_precise_u8x64(self, a: u8x64<Self>, indices: u8x64<Self>) -> u8x64<Self> {
let bytes = Bytes::to_bytes(a);
let mut output = [0u8; 64usize];
for lane in 0..64usize {
let index = indices[lane] as usize;
let value = bytes[index % 64usize];
output[lane] = if index < 64usize { value } else { 0 };
}
let result: u8x64<Self> = output.simd_into(self);
Bytes::from_bytes(result)
}
#[inline(always)]
fn split_u8x64(self, a: u8x64<Self>) -> (u8x32<Self>, u8x32<Self>) {
(
u8x32 {
val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
simd: self,
},
u8x32 {
val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
simd: self,
},
)
}
#[inline(always)]
fn set_mask8x64(self, a: &mut mask8x64<Self>, index: usize, value: bool) -> () {
assert!(
index < 64usize,
"mask lane index {index} is out of bounds for {} lanes",
64usize
);
let mut lanes: [i8; 64usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn split_mask8x64(self, a: mask8x64<Self>) -> (mask8x32<Self>, mask8x32<Self>) {
(
mask8x32 {
val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
simd: self,
},
mask8x32 {
val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
simd: self,
},
)
}
#[inline(always)]
fn slide_i16x32<const SHIFT: usize>(self, a: i16x32<Self>, b: i16x32<Self>) -> i16x32<Self> {
if SHIFT >= 32usize {
return b;
}
let result = cross_block_slide_128x4(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 2usize,
);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_i16x32(self, a: i16x32<Self>) -> (i16x16<Self>, i16x16<Self>) {
(
i16x16 {
val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
simd: self,
},
i16x16 {
val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
simd: self,
},
)
}
#[inline(always)]
fn slide_u16x32<const SHIFT: usize>(self, a: u16x32<Self>, b: u16x32<Self>) -> u16x32<Self> {
if SHIFT >= 32usize {
return b;
}
let result = cross_block_slide_128x4(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 2usize,
);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_u16x32(self, a: u16x32<Self>) -> (u16x16<Self>, u16x16<Self>) {
(
u16x16 {
val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
simd: self,
},
u16x16 {
val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
simd: self,
},
)
}
#[inline(always)]
fn set_mask16x32(self, a: &mut mask16x32<Self>, index: usize, value: bool) -> () {
assert!(
index < 32usize,
"mask lane index {index} is out of bounds for {} lanes",
32usize
);
let mut lanes: [i16; 32usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn split_mask16x32(self, a: mask16x32<Self>) -> (mask16x16<Self>, mask16x16<Self>) {
(
mask16x16 {
val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
simd: self,
},
mask16x16 {
val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
simd: self,
},
)
}
#[inline(always)]
fn slide_i32x16<const SHIFT: usize>(self, a: i32x16<Self>, b: i32x16<Self>) -> i32x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = cross_block_slide_128x4(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_i32x16(self, a: i32x16<Self>) -> (i32x8<Self>, i32x8<Self>) {
(
i32x8 {
val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
simd: self,
},
i32x8 {
val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
simd: self,
},
)
}
#[inline(always)]
fn slide_u32x16<const SHIFT: usize>(self, a: u32x16<Self>, b: u32x16<Self>) -> u32x16<Self> {
if SHIFT >= 16usize {
return b;
}
let result = cross_block_slide_128x4(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 4usize,
);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_u32x16(self, a: u32x16<Self>) -> (u32x8<Self>, u32x8<Self>) {
(
u32x8 {
val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
simd: self,
},
u32x8 {
val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
simd: self,
},
)
}
#[inline(always)]
fn set_mask32x16(self, a: &mut mask32x16<Self>, index: usize, value: bool) -> () {
assert!(
index < 16usize,
"mask lane index {index} is out of bounds for {} lanes",
16usize
);
let mut lanes: [i32; 16usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn split_mask32x16(self, a: mask32x16<Self>) -> (mask32x8<Self>, mask32x8<Self>) {
(
mask32x8 {
val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
simd: self,
},
mask32x8 {
val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
simd: self,
},
)
}
#[inline(always)]
fn slide_f64x8<const SHIFT: usize>(self, a: f64x8<Self>, b: f64x8<Self>) -> f64x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = cross_block_slide_128x4(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_f64x8(self, a: f64x8<Self>) -> (f64x4<Self>, f64x4<Self>) {
(
f64x4 {
val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
simd: self,
},
f64x4 {
val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
simd: self,
},
)
}
#[inline(always)]
fn slide_i64x8<const SHIFT: usize>(self, a: i64x8<Self>, b: i64x8<Self>) -> i64x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = cross_block_slide_128x4(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_i64x8(self, a: i64x8<Self>) -> (i64x4<Self>, i64x4<Self>) {
(
i64x4 {
val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
simd: self,
},
i64x4 {
val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
simd: self,
},
)
}
#[inline(always)]
fn slide_u64x8<const SHIFT: usize>(self, a: u64x8<Self>, b: u64x8<Self>) -> u64x8<Self> {
if SHIFT >= 8usize {
return b;
}
let result = cross_block_slide_128x4(
Bytes::to_bytes(a).val.0,
Bytes::to_bytes(b).val.0,
SHIFT * 8usize,
);
Bytes::from_bytes(u8x64 {
val: crate::support::Aligned512(result),
simd: self,
})
}
#[inline(always)]
fn split_u64x8(self, a: u64x8<Self>) -> (u64x4<Self>, u64x4<Self>) {
(
u64x4 {
val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
simd: self,
},
u64x4 {
val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
simd: self,
},
)
}
#[inline(always)]
fn set_mask64x8(self, a: &mut mask64x8<Self>, index: usize, value: bool) -> () {
assert!(
index < 8usize,
"mask lane index {index} is out of bounds for {} lanes",
8usize
);
let mut lanes: [i64; 8usize] = (*a).into();
lanes[index] = if value { !0 } else { 0 };
*a = lanes.simd_into(self);
}
#[inline(always)]
fn split_mask64x8(self, a: mask64x8<Self>) -> (mask64x4<Self>, mask64x4<Self>) {
(
mask64x4 {
val: crate::support::Aligned256([a.val.0[0], a.val.0[1]]),
simd: self,
},
mask64x4 {
val: crate::support::Aligned256([a.val.0[2], a.val.0[3]]),
simd: self,
},
)
}
}
impl<S: Simd> SimdFrom<v128, S> for f32x4<S> {
#[inline(always)]
fn simd_from(simd: S, arch: v128) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<f32x4<S>> for v128 {
#[inline(always)]
fn from(value: f32x4<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<v128, S> for i8x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: v128) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i8x16<S>> for v128 {
#[inline(always)]
fn from(value: i8x16<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<v128, S> for u8x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: v128) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u8x16<S>> for v128 {
#[inline(always)]
fn from(value: u8x16<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<v128, S> for mask8x16<S> {
#[inline(always)]
fn simd_from(simd: S, arch: v128) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<mask8x16<S>> for v128 {
#[inline(always)]
fn from(value: mask8x16<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<v128, S> for i16x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: v128) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i16x8<S>> for v128 {
#[inline(always)]
fn from(value: i16x8<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<v128, S> for u16x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: v128) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u16x8<S>> for v128 {
#[inline(always)]
fn from(value: u16x8<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<v128, S> for mask16x8<S> {
#[inline(always)]
fn simd_from(simd: S, arch: v128) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<mask16x8<S>> for v128 {
#[inline(always)]
fn from(value: mask16x8<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<v128, S> for i32x4<S> {
#[inline(always)]
fn simd_from(simd: S, arch: v128) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i32x4<S>> for v128 {
#[inline(always)]
fn from(value: i32x4<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<v128, S> for u32x4<S> {
#[inline(always)]
fn simd_from(simd: S, arch: v128) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u32x4<S>> for v128 {
#[inline(always)]
fn from(value: u32x4<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<v128, S> for mask32x4<S> {
#[inline(always)]
fn simd_from(simd: S, arch: v128) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<mask32x4<S>> for v128 {
#[inline(always)]
fn from(value: mask32x4<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<v128, S> for f64x2<S> {
#[inline(always)]
fn simd_from(simd: S, arch: v128) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<f64x2<S>> for v128 {
#[inline(always)]
fn from(value: f64x2<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<v128, S> for i64x2<S> {
#[inline(always)]
fn simd_from(simd: S, arch: v128) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<i64x2<S>> for v128 {
#[inline(always)]
fn from(value: i64x2<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<v128, S> for u64x2<S> {
#[inline(always)]
fn simd_from(simd: S, arch: v128) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<u64x2<S>> for v128 {
#[inline(always)]
fn from(value: u64x2<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
impl<S: Simd> SimdFrom<v128, S> for mask64x2<S> {
#[inline(always)]
fn simd_from(simd: S, arch: v128) -> Self {
Self {
val: crate::transmute::checked_transmute_copy(&arch),
simd,
}
}
}
impl<S: Simd> From<mask64x2<S>> for v128 {
#[inline(always)]
fn from(value: mask64x2<S>) -> Self {
crate::transmute::checked_transmute_copy(&value.val)
}
}
#[doc = r" This is a vector extend, like `vext` on ARM or `alignr` on x86, that takes a non-const shift argument."]
#[doc = r" The shift is still expected to be constant in practice, so the match statement will be optimized out."]
#[doc = r" This exists because Rust doesn't currently let you do math on const generics."]
#[inline(always)]
fn dyn_slide_128(a: v128, b: v128, shift: usize) -> v128 {
match shift {
0 => i8x16_shuffle::<0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15>(a, b),
1 => i8x16_shuffle::<1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16>(a, b),
2 => i8x16_shuffle::<2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17>(a, b),
3 => i8x16_shuffle::<3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18>(a, b),
4 => i8x16_shuffle::<4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19>(a, b),
5 => i8x16_shuffle::<5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20>(a, b),
6 => i8x16_shuffle::<6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21>(a, b),
7 => i8x16_shuffle::<7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22>(a, b),
8 => i8x16_shuffle::<8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23>(a, b),
9 => i8x16_shuffle::<9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24>(a, b),
10 => i8x16_shuffle::<10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25>(a, b),
11 => i8x16_shuffle::<11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26>(a, b),
12 => i8x16_shuffle::<12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27>(a, b),
13 => i8x16_shuffle::<13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28>(a, b),
14 => i8x16_shuffle::<14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29>(a, b),
15 => i8x16_shuffle::<15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30>(a, b),
_ => unreachable!(),
}
}
#[doc = r" Concatenates `a` and `b` (each N blocks) and extracts N blocks starting at byte offset `shift_bytes`."]
#[inline(always)]
fn cross_block_slide_128x2(
a: [v128; 2usize],
b: [v128; 2usize],
shift_bytes: usize,
) -> [v128; 2usize] {
[
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(&a, &b, 0usize, shift_bytes);
dyn_slide_128(lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(&a, &b, 1usize, shift_bytes);
dyn_slide_128(lo, hi, shift_bytes % 16)
},
]
}
#[doc = r" Concatenates `a` and `b` (each N blocks) and extracts N blocks starting at byte offset `shift_bytes`."]
#[inline(always)]
fn cross_block_slide_128x4(
a: [v128; 4usize],
b: [v128; 4usize],
shift_bytes: usize,
) -> [v128; 4usize] {
[
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(&a, &b, 0usize, shift_bytes);
dyn_slide_128(lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(&a, &b, 1usize, shift_bytes);
dyn_slide_128(lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(&a, &b, 2usize, shift_bytes);
dyn_slide_128(lo, hi, shift_bytes % 16)
},
{
let [lo, hi] = crate::support::cross_block_slide_blocks_at(&a, &b, 3usize, shift_bytes);
dyn_slide_128(lo, hi, shift_bytes % 16)
},
]
}