use crate::ValidLayout;
use crate::detail::DataKind;
use crate::mask::{Mask, MaskLayout};
use crate::num::SimdInt;
use crate::simd::{Simd, SimdLayout, SimdScalar};
use multitype::Int;
#[cfg(target_arch = "x86")]
#[allow(unused_imports)]
use core::arch::x86::*;
#[cfg(target_arch = "x86_64")]
#[allow(unused_imports)]
use core::arch::x86_64::*;
impl<T, const N: usize> crate::num::seal::SimdInt for Simd<T, N>
where
T: SimdScalar,
SimdLayout<T, N>: ValidLayout,
{}
impl<T, const N: usize> SimdInt<T, N> for Simd<T, N>
where
T: SimdScalar + Int<Uint: SimdScalar>,
SimdLayout<T, N>: ValidLayout,
{
const BITS: u32 = T::BITS * N as u32;
const MIN: Self = Self::splat(T::MIN);
const MAX: Self = Self::splat(T::MAX);
const ZERO: Self = Self::splat(T::ZERO);
const ONE: Self = Self::splat(T::ONE);
#[inline]
fn from_le(value: Self) -> Self {
value.map(T::from_le)
}
#[inline]
fn from_be(value: Self) -> Self {
value.map(T::from_le)
}
#[inline]
fn div_euclid(self, rhs: Self) -> Self {
self.zip_with(rhs, T::div_euclid)
}
#[inline]
fn rem_euclid(self, rhs: Self) -> Self {
self.zip_with(rhs, T::rem_euclid)
}
#[inline]
fn rotate_left(mut self, rhs: Simd<u32, N>) -> Self
where
SimdLayout<u32, N>: ValidLayout,
{
for i in 0..N {
let lhs = &mut self[i];
let rhs = rhs[i];
*lhs = lhs.rotate_left(rhs);
}
self
}
#[inline]
fn rotate_right(mut self, rhs: Simd<u32, N>) -> Self
where
SimdLayout<u32, N>: ValidLayout,
{
for i in 0..N {
let lhs = &mut self[i];
let rhs = rhs[i];
*lhs = lhs.rotate_right(rhs);
}
self
}
#[inline]
fn swap_bytes(self) -> Self {
self.map(T::swap_bytes)
}
#[inline]
fn reverse_bits(self) -> Self {
self.map(T::reverse_bits)
}
#[inline]
fn abs(self) -> Self {
self.map(T::abs)
}
#[inline]
fn unsigned_abs(self) -> Simd<T::Uint, N>
where
SimdLayout<T::Uint, N>: ValidLayout,
{
self.map(T::unsigned_abs)
}
#[inline]
fn pow(mut self, rhs: Simd<u32, N>) -> Self
where
SimdLayout<u32, N>: ValidLayout,
{
for i in 0..N {
let lhs = &mut self[i];
let rhs = rhs[i];
*lhs = lhs.pow(rhs);
}
self
}
#[inline]
fn isqrt(self) -> Self {
self.map(T::isqrt)
}
#[inline]
fn ilog(self, base: Self) -> Simd<u32, N>
where
SimdLayout<u32, N>: ValidLayout,
{
let mut result = Simd::splat(0);
for i in 0..N {
let result = &mut result[i];
let lhs = self[i];
let base = base[i];
*result = lhs.ilog(base);
}
result
}
#[inline]
fn ilog2(self) -> Simd<u32, N>
where
SimdLayout<u32, N>: ValidLayout,
{
self.map(T::ilog2)
}
#[inline]
fn ilog10(self) -> Simd<u32, N>
where
SimdLayout<u32, N>: ValidLayout,
{
self.map(T::ilog10)
}
#[inline]
fn abs_diff(self, rhs: Self) -> Simd<T::Uint, N>
where
SimdLayout<T::Uint, N>: ValidLayout,
{
let mut result = Simd::<T::Uint, N>::default();
for i in 0..N {
let result = &mut result[i];
let lhs = self[i];
let rhs = rhs[i];
*result = lhs.abs_diff(rhs);
}
result
}
#[inline]
fn midpoint(self, rhs: Self) -> Self {
self.zip_with(rhs, T::midpoint)
}
#[inline]
fn signum(self) -> Self {
self.map(T::signum)
}
#[inline]
fn is_negative(self) -> Mask<N>
where
MaskLayout<N>: ValidLayout,
{
self.to_array().map(T::is_negative).into()
}
#[inline]
fn is_positive(self) -> Mask<N>
where
MaskLayout<N>: ValidLayout,
{
self.to_array().map(T::is_positive).into()
}
#[inline]
fn count_zeros(self) -> Simd<u32, N>
where
SimdLayout<u32, N>: ValidLayout,
{
self.map(T::count_zeros)
}
#[inline]
fn count_ones(self) -> Simd<u32, N>
where
SimdLayout<u32, N>: ValidLayout,
{
self.map(T::count_ones)
}
#[inline]
fn leading_zeros(self) -> Simd<u32, N>
where
SimdLayout<u32, N>: ValidLayout,
{
self.map(T::leading_zeros)
}
#[inline]
fn leading_ones(self) -> Simd<u32, N>
where
SimdLayout<u32, N>: ValidLayout,
{
self.map(T::leading_ones)
}
#[inline]
fn trailing_zeros(self) -> Simd<u32, N>
where
SimdLayout<u32, N>: ValidLayout,
{
self.map(T::trailing_zeros)
}
#[inline]
fn trailing_ones(self) -> Simd<u32, N>
where
SimdLayout<u32, N>: ValidLayout,
{
self.map(T::trailing_ones)
}
#[inline]
#[track_caller]
fn strict_add(self, rhs: Self) -> Self {
self.zip_with(rhs, T::strict_add)
}
#[inline]
#[track_caller]
fn strict_add_unsigned(self, rhs: Simd<T::Uint, N>) -> Self
where
SimdLayout<T::Uint, N>: ValidLayout,
{
self.zip_with(rhs, T::strict_add_unsigned)
}
#[inline]
#[track_caller]
fn strict_sub(self, rhs: Self) -> Self {
self.zip_with(rhs, T::strict_sub)
}
#[inline]
#[track_caller]
fn strict_sub_unsigned(self, rhs: Simd<T::Uint, N>) -> Self
where
SimdLayout<T::Uint, N>: ValidLayout,
{
self.zip_with(rhs, T::strict_sub_unsigned)
}
#[inline]
#[track_caller]
fn strict_mul(self, rhs: Self) -> Self {
self.zip_with(rhs, T::strict_mul)
}
#[inline]
#[track_caller]
fn strict_div(self, rhs: Self) -> Self {
self.zip_with(rhs, T::strict_div)
}
#[inline]
#[track_caller]
fn strict_div_euclid(self, rhs: Self) -> Self {
self.zip_with(rhs, T::strict_div_euclid)
}
#[inline]
#[track_caller]
fn strict_rem(self, rhs: Self) -> Self {
self.zip_with(rhs, T::strict_rem)
}
#[inline]
#[track_caller]
fn strict_rem_euclid(self, rhs: Self) -> Self {
self.zip_with(rhs, T::strict_rem_euclid)
}
#[inline]
#[track_caller]
fn strict_shl(self, rhs: Simd<u32, N>) -> Self
where
SimdLayout<u32, N>: ValidLayout,
{
self.zip_with(rhs, T::strict_shl)
}
#[inline]
#[track_caller]
fn strict_shr(self, rhs: Simd<u32, N>) -> Self
where
SimdLayout<u32, N>: ValidLayout,
{
self.zip_with(rhs, T::strict_shr)
}
#[inline]
#[track_caller]
fn strict_neg(self) -> Self {
self.map(T::strict_neg)
}
#[inline]
#[track_caller]
fn strict_abs(self) -> Self {
self.map(T::strict_abs)
}
#[inline]
#[track_caller]
fn strict_pow(self, rhs: Simd<u32, N>) -> Self
where
SimdLayout<u32, N>: ValidLayout,
{
self.zip_with(rhs, T::strict_pow)
}
#[inline]
#[track_caller]
unsafe fn unchecked_add(self, rhs: Self) -> Self {
self.wrapping_add(rhs)
}
#[inline]
#[track_caller]
unsafe fn unchecked_sub(self, rhs: Self) -> Self {
self.wrapping_sub(rhs)
}
#[inline]
#[track_caller]
unsafe fn unchecked_mul(self, rhs: Self) -> Self {
self.wrapping_mul(rhs)
}
#[inline]
fn overflowing_add(mut self, rhs: Self) -> (Self, Mask<N>)
where
MaskLayout<N>: ValidLayout,
{
let mut overflow = [false; N];
for i in 0..N {
let lhs = &mut self [i];
let overflow = &mut overflow[i];
let rhs = rhs[i];
(*lhs, *overflow) = lhs.overflowing_add(rhs);
}
(self, overflow.into())
}
#[inline]
fn overflowing_add_unsigned(mut self, rhs: Simd<T::Uint, N>) -> (Self, Mask<N>)
where
SimdLayout<T::Uint, N>: ValidLayout,
MaskLayout<N>: ValidLayout,
{
let mut overflow = [false; N];
for i in 0..N {
let lhs = &mut self [i];
let overflow = &mut overflow[i];
let rhs = rhs[i];
(*lhs, *overflow) = lhs.overflowing_add_unsigned(rhs);
}
(self, overflow.into())
}
#[inline]
fn overflowing_sub(mut self, rhs: Self) -> (Self, Mask<N>)
where
MaskLayout<N>: ValidLayout,
{
let mut overflow = [false; N];
for i in 0..N {
let lhs = &mut self [i];
let overflow = &mut overflow[i];
let rhs = rhs[i];
(*lhs, *overflow) = lhs.overflowing_sub(rhs);
}
(self, overflow.into())
}
#[inline]
fn overflowing_sub_unsigned(mut self, rhs: Simd<T::Uint, N>) -> (Self, Mask<N>)
where
SimdLayout<T::Uint, N>: ValidLayout,
MaskLayout<N>: ValidLayout,
{
let mut overflow = [false; N];
for i in 0..N {
let lhs = &mut self [i];
let overflow = &mut overflow[i];
let rhs = rhs[i];
(*lhs, *overflow) = lhs.overflowing_sub_unsigned(rhs);
}
(self, overflow.into())
}
#[inline]
fn overflowing_mul(mut self, rhs: Self) -> (Self, Mask<N>)
where
MaskLayout<N>: ValidLayout,
{
let mut overflow = [false; N];
for i in 0..N {
let lhs = &mut self [i];
let overflow = &mut overflow[i];
let rhs = rhs[i];
(*lhs, *overflow) = lhs.overflowing_mul(rhs);
}
(self, overflow.into())
}
#[inline]
fn overflowing_div(mut self, rhs: Self) -> (Self, Mask<N>)
where
MaskLayout<N>: ValidLayout,
{
let mut overflow = [false; N];
for i in 0..N {
let lhs = &mut self [i];
let overflow = &mut overflow[i];
let rhs = rhs[i];
(*lhs, *overflow) = lhs.overflowing_div(rhs);
}
(self, overflow.into())
}
#[inline]
fn overflowing_div_euclid(mut self, rhs: Self) -> (Self, Mask<N>)
where
MaskLayout<N>: ValidLayout,
{
let mut overflow = [false; N];
for i in 0..N {
let lhs = &mut self [i];
let overflow = &mut overflow[i];
let rhs = rhs[i];
(*lhs, *overflow) = lhs.overflowing_div_euclid(rhs);
}
(self, overflow.into())
}
#[inline]
fn overflowing_rem(mut self, rhs: Self) -> (Self, Mask<N>)
where
MaskLayout<N>: ValidLayout,
{
let mut overflow = [false; N];
for i in 0..N {
let lhs = &mut self [i];
let overflow = &mut overflow[i];
let rhs = rhs[i];
(*lhs, *overflow) = lhs.overflowing_rem(rhs);
}
(self, overflow.into())
}
#[inline]
fn overflowing_rem_euclid(mut self, rhs: Self) -> (Self, Mask<N>)
where
MaskLayout<N>: ValidLayout,
{
let mut overflow = [false; N];
for i in 0..N {
let lhs = &mut self [i];
let overflow = &mut overflow[i];
let rhs = rhs[i];
(*lhs, *overflow) = lhs.overflowing_rem_euclid(rhs);
}
(self, overflow.into())
}
#[inline]
fn overflowing_shl(mut self, rhs: Simd<u32, N>) -> (Self, Mask<N>)
where
SimdLayout<u32, N>: ValidLayout,
MaskLayout<N>: ValidLayout,
{
let mut overflow = [false; N];
for i in 0..N {
let lhs = &mut self [i];
let overflow = &mut overflow[i];
let rhs = rhs[i];
(*lhs, *overflow) = lhs.overflowing_shl(rhs);
}
(self, overflow.into())
}
#[inline]
fn overflowing_shr(mut self, rhs: Simd<u32, N>) -> (Self, Mask<N>)
where
SimdLayout<u32, N>: ValidLayout,
MaskLayout<N>: ValidLayout,
{
let mut overflow = [false; N];
for i in 0..N {
let lhs = &mut self [i];
let overflow = &mut overflow[i];
let rhs = rhs[i];
(*lhs, *overflow) = lhs.overflowing_shr(rhs);
}
(self, overflow.into())
}
#[inline]
fn overflowing_neg(mut self) -> (Self, Mask<N>)
where
MaskLayout<N>: ValidLayout,
{
let mut overflow = [false; N];
for i in 0..N {
let lhs = &mut self [i];
let overflow = &mut overflow[i];
(*lhs, *overflow) = lhs.overflowing_neg();
}
(self, overflow.into())
}
#[inline]
fn overflowing_abs(mut self) -> (Self, Mask<N>)
where
MaskLayout<N>: ValidLayout,
{
let mut overflow = [false; N];
for i in 0..N {
let lhs = &mut self [i];
let overflow = &mut overflow[i];
(*lhs, *overflow) = lhs.overflowing_abs();
}
(self, overflow.into())
}
#[inline]
fn overflowing_pow(mut self, rhs: Simd<u32, N>) -> (Self, Mask<N>)
where
SimdLayout<u32, N>: ValidLayout,
MaskLayout<N>: ValidLayout,
{
let mut overflow = [false; N];
for i in 0..N {
let lhs = &mut self [i];
let overflow = &mut overflow[i];
let rhs = rhs[i];
(*lhs, *overflow) = lhs.overflowing_pow(rhs);
}
(self, overflow.into())
}
#[inline]
fn wrapping_add(mut self, rhs: Self) -> Self {
match (T::DATA_KIND, N) {
#[cfg(target_feature = "sse2")]
(DataKind::I8, 16) => unsafe {
let lhs = self.to_native();
let rhs = rhs.to_native();
let result = _mm_add_epi8(lhs, rhs);
Self::from_native(result)
}
#[cfg(target_feature = "sse2")]
(DataKind::I16, 8) => unsafe {
let lhs = self.to_native();
let rhs = rhs.to_native();
let result = _mm_add_epi16(lhs, rhs);
Self::from_native(result)
}
#[cfg(target_feature = "sse2")]
(DataKind::I32, 4) => unsafe {
let lhs = self.to_native();
let rhs = rhs.to_native();
let result = _mm_add_epi32(lhs, rhs);
Self::from_native(result)
}
#[cfg(target_feature = "sse2")]
(DataKind::I64, 2) => unsafe {
let lhs = self.to_native();
let rhs = rhs.to_native();
let result = _mm_add_epi64(lhs, rhs);
Self::from_native(result)
}
_ => {
for i in 0..N {
let lhs = &mut self[i];
let rhs = rhs[i];
*lhs = lhs.wrapping_add(rhs);
}
self
}
}
}
#[inline]
fn wrapping_add_unsigned(mut self, rhs: Simd<T::Uint, N>) -> Self
where
SimdLayout<T::Uint, N>: ValidLayout,
{
match (T::DATA_KIND, N) {
#[cfg(target_feature = "sse2")]
(DataKind::I8, 16) => unsafe {
let lhs = self.to_native();
let rhs = rhs.to_native();
let result = _mm_add_epi8(lhs, rhs);
Self::from_native(result)
}
#[cfg(target_feature = "sse2")]
(DataKind::I16, 8) => unsafe {
let lhs = self.to_native();
let rhs = rhs.to_native();
let result = _mm_add_epi16(lhs, rhs);
Self::from_native(result)
}
#[cfg(target_feature = "sse2")]
(DataKind::I32, 4) => unsafe {
let lhs = self.to_native();
let rhs = rhs.to_native();
let result = _mm_add_epi32(lhs, rhs);
Self::from_native(result)
}
#[cfg(target_feature = "sse2")]
(DataKind::I64, 2) => unsafe {
let lhs = self.to_native();
let rhs = rhs.to_native();
let result = _mm_add_epi64(lhs, rhs);
Self::from_native(result)
}
_ => {
for i in 0..N {
let lhs = &mut self[i];
let rhs = rhs[i];
*lhs = lhs.wrapping_add_unsigned(rhs);
}
self
}
}
}
#[inline]
fn wrapping_sub(mut self, rhs: Self) -> Self {
match (T::DATA_KIND, N) {
#[cfg(target_feature = "sse2")]
(DataKind::I8, 16) => unsafe {
let lhs = self.to_native();
let rhs = rhs.to_native();
let result = _mm_sub_epi8(lhs, rhs);
Self::from_native(result)
}
#[cfg(target_feature = "sse2")]
(DataKind::I16, 8) => unsafe {
let lhs = self.to_native();
let rhs = rhs.to_native();
let result = _mm_sub_epi16(lhs, rhs);
Self::from_native(result)
}
#[cfg(target_feature = "sse2")]
(DataKind::I32, 4) => unsafe {
let lhs = self.to_native();
let rhs = rhs.to_native();
let result = _mm_sub_epi32(lhs, rhs);
Self::from_native(result)
}
#[cfg(target_feature = "sse2")]
(DataKind::I64, 2) => unsafe {
let lhs = self.to_native();
let rhs = rhs.to_native();
let result = _mm_sub_epi64(lhs, rhs);
Self::from_native(result)
}
_ => {
for i in 0..N {
let lhs = &mut self[i];
let rhs = rhs[i];
*lhs = lhs.wrapping_sub(rhs);
}
self
}
}
}
#[inline]
fn wrapping_sub_unsigned(mut self, rhs: Simd<T::Uint, N>) -> Self
where
SimdLayout<T::Uint, N>: ValidLayout,
{
match (T::DATA_KIND, N) {
#[cfg(target_feature = "sse2")]
(DataKind::I8, 16) => unsafe {
let lhs = self.to_native();
let rhs = rhs.to_native();
let result = _mm_sub_epi8(lhs, rhs);
Self::from_native(result)
}
#[cfg(target_feature = "sse2")]
(DataKind::I16, 8) => unsafe {
let lhs = self.to_native();
let rhs = rhs.to_native();
let result = _mm_sub_epi16(lhs, rhs);
Self::from_native(result)
}
#[cfg(target_feature = "sse2")]
(DataKind::I32, 4) => unsafe {
let lhs = self.to_native();
let rhs = rhs.to_native();
let result = _mm_sub_epi32(lhs, rhs);
Self::from_native(result)
}
#[cfg(target_feature = "sse2")]
(DataKind::I64, 2) => unsafe {
let lhs = self.to_native();
let rhs = rhs.to_native();
let result = _mm_sub_epi64(lhs, rhs);
Self::from_native(result)
}
_ => {
for i in 0..N {
let lhs = &mut self[i];
let rhs = rhs[i];
*lhs = lhs.wrapping_sub_unsigned(rhs);
}
self
}
}
}
#[inline]
fn wrapping_mul(self, rhs: Self) -> Self {
self.zip_with(rhs, T::wrapping_mul)
}
#[inline]
fn wrapping_div(self, rhs: Self) -> Self {
self.zip_with(rhs, T::wrapping_div)
}
#[inline]
fn wrapping_div_euclid(self, rhs: Self) -> Self {
self.zip_with(rhs, T::wrapping_div_euclid)
}
#[inline]
fn wrapping_rem(self, rhs: Self) -> Self {
self.zip_with(rhs, T::wrapping_rem)
}
#[inline]
fn wrapping_rem_euclid(self, rhs: Self) -> Self {
self.zip_with(rhs, T::wrapping_rem_euclid)
}
#[inline]
fn wrapping_shl(mut self, rhs: Simd<u32, N>) -> Self
where
SimdLayout<u32, N>: ValidLayout,
{
for i in 0..N {
let lhs = &mut self[i];
let rhs = rhs[i];
*lhs = lhs.wrapping_shl(rhs);
}
self
}
#[inline]
fn wrapping_shr(mut self, rhs: Simd<u32, N>) -> Self
where
SimdLayout<u32, N>: ValidLayout,
{
for i in 0..N {
let lhs = &mut self[i];
let rhs = rhs[i];
*lhs = lhs.wrapping_shr(rhs);
}
self
}
#[inline]
fn wrapping_neg(self) -> Self {
self.map(T::wrapping_neg)
}
#[inline]
fn wrapping_abs(self) -> Self {
self.map(T::wrapping_abs)
}
#[inline]
fn wrapping_pow(self, rhs: Simd<u32, N>) -> Self
where
SimdLayout<u32, N>: ValidLayout,
{
self.zip_with(rhs, T::wrapping_pow)
}
#[inline]
fn saturating_add(self, rhs: Self) -> Self {
match (T::DATA_KIND, N) {
#[cfg(target_feature = "sse2")]
(DataKind::I8, 16) => unsafe {
let lhs = self.to_native();
let rhs = rhs.to_native();
let result = _mm_adds_epi8(lhs, rhs);
Self::from_native(result)
}
#[cfg(target_feature = "sse2")]
(DataKind::I16, 8) => unsafe {
let lhs = self.to_native();
let rhs = rhs.to_native();
let result = _mm_adds_epi16(lhs, rhs);
Self::from_native(result)
}
_ => {
self.zip_with(rhs, T::saturating_add)
}
}
}
#[inline]
fn saturating_add_unsigned(self, rhs: Simd<T::Uint, N>) -> Self
where
SimdLayout<T::Uint, N>: ValidLayout,
{
self.zip_with(rhs, T::saturating_add_unsigned)
}
#[inline]
fn saturating_sub(mut self, rhs: Self) -> Self {
match (T::DATA_KIND, N) {
#[cfg(target_feature = "sse2")]
(DataKind::I8, 16) => unsafe {
let lhs = self.to_native();
let rhs = rhs.to_native();
let result = _mm_subs_epi8(lhs, rhs);
Self::from_native(result)
}
#[cfg(target_feature = "sse2")]
(DataKind::I16, 8) => unsafe {
let lhs = self.to_native();
let rhs = rhs.to_native();
let result = _mm_subs_epi16(lhs, rhs);
Self::from_native(result)
}
_ => {
for i in 0..N {
let lhs = &mut self[i];
let rhs = rhs[i];
*lhs = lhs.saturating_sub(rhs);
}
self
}
}
}
#[inline]
fn saturating_sub_unsigned(self, rhs: Simd<T::Uint, N>) -> Self
where
SimdLayout<T::Uint, N>: ValidLayout,
{
self.zip_with(rhs, T::saturating_sub_unsigned)
}
#[inline]
fn saturating_mul(self, rhs: Self) -> Self {
self.zip_with(rhs, T::saturating_mul)
}
#[inline]
fn saturating_div(self, rhs: Self) -> Self {
self.zip_with(rhs, T::saturating_div)
}
#[inline]
fn saturating_neg(self) -> Self {
self.map(T::saturating_neg)
}
#[inline]
fn saturating_abs(self) -> Self {
self.map(T::saturating_abs)
}
#[inline]
fn saturating_pow(mut self, rhs: Simd<u32, N>) -> Self
where
SimdLayout<u32, N>: ValidLayout,
{
for i in 0..N {
let lhs = &mut self[i];
let rhs = rhs[i];
*lhs = lhs.saturating_pow(rhs);
}
self
}
#[inline]
fn unbounded_shl(self, rhs: Simd<u32, N>) -> Self
where
SimdLayout<u32, N>: ValidLayout,
{
self.zip_with(rhs, T::unbounded_shl)
}
#[inline]
fn unbounded_shr(self, rhs: Simd<u32, N>) -> Self
where
SimdLayout<u32, N>: ValidLayout,
{
self.zip_with(rhs, T::unbounded_shr)
}
#[inline(always)]
fn cast_unsigned(self) -> Simd<T::Uint, N>
where
SimdLayout<T::Uint, N>: ValidLayout,
{
unsafe { self.transmute() }
}
#[inline]
fn to_le(self) -> Self {
self.map(T::to_le)
}
#[inline]
fn to_be(self) -> Self {
self.map(T::to_be)
}
}