pub(crate) trait WideningMultiply<RHS = Self> {
type Output;
fn wmul(self, x: RHS) -> Self::Output;
}
macro_rules! wmul_impl {
($ty:ty, $wide:ty, $shift:expr) => {
impl WideningMultiply for $ty {
type Output = ($ty, $ty);
#[inline(always)]
fn wmul(self, x: $ty) -> Self::Output {
let tmp = (self as $wide) * (x as $wide);
((tmp >> $shift) as $ty, tmp as $ty)
}
}
};
($(($ty:ident, $wide:ident),)+, $shift:expr) => {
$(
impl WideningMultiply for $ty {
type Output = ($ty, $ty);
#[inline(always)]
fn wmul(self, x: $ty) -> Self::Output {
let y: $wide = self.cast();
let x: $wide = x.cast();
let tmp = y * x;
let hi: $ty = (tmp >> $shift).cast();
let lo: $ty = tmp.cast();
(hi, lo)
}
}
)+
};
}
wmul_impl! { u8, u16, 8 }
wmul_impl! { u16, u32, 16 }
wmul_impl! { u32, u64, 32 }
wmul_impl! { u64, u128, 64 }
macro_rules! wmul_impl_large {
($ty:ty, $half:expr) => {
impl WideningMultiply for $ty {
type Output = ($ty, $ty);
#[inline(always)]
fn wmul(self, b: $ty) -> Self::Output {
const LOWER_MASK: $ty = !0 >> $half;
let mut low = (self & LOWER_MASK).wrapping_mul(b & LOWER_MASK);
let mut t = low >> $half;
low &= LOWER_MASK;
t += (self >> $half).wrapping_mul(b & LOWER_MASK);
low += (t & LOWER_MASK) << $half;
let mut high = t >> $half;
t = low >> $half;
low &= LOWER_MASK;
t += (b >> $half).wrapping_mul(self & LOWER_MASK);
low += (t & LOWER_MASK) << $half;
high += t >> $half;
high += (self >> $half).wrapping_mul(b >> $half);
(high, low)
}
}
};
(($($ty:ty,)+) $scalar:ty, $half:expr) => {
$(
impl WideningMultiply for $ty {
type Output = ($ty, $ty);
#[inline(always)]
fn wmul(self, b: $ty) -> Self::Output {
const LOWER_MASK: $scalar = !0 >> $half;
let mut low = (self & LOWER_MASK) * (b & LOWER_MASK);
let mut t = low >> $half;
low &= LOWER_MASK;
t += (self >> $half) * (b & LOWER_MASK);
low += (t & LOWER_MASK) << $half;
let mut high = t >> $half;
t = low >> $half;
low &= LOWER_MASK;
t += (b >> $half) * (self & LOWER_MASK);
low += (t & LOWER_MASK) << $half;
high += t >> $half;
high += (self >> $half) * (b >> $half);
(high, low)
}
}
)+
};
}
wmul_impl_large! { u128, 64 }
macro_rules! wmul_impl_usize {
($ty:ty) => {
impl WideningMultiply for usize {
type Output = (usize, usize);
#[inline(always)]
fn wmul(self, x: usize) -> Self::Output {
let (high, low) = (self as $ty).wmul(x as $ty);
(high as usize, low as usize)
}
}
};
}
#[cfg(target_pointer_width = "16")]
wmul_impl_usize! { u16 }
#[cfg(target_pointer_width = "32")]
wmul_impl_usize! { u32 }
#[cfg(target_pointer_width = "64")]
wmul_impl_usize! { u64 }
pub(crate) trait FloatSIMDUtils {
fn all_lt(self, other: Self) -> bool;
fn all_le(self, other: Self) -> bool;
fn all_finite(self) -> bool;
type Mask;
fn finite_mask(self) -> Self::Mask;
fn gt_mask(self, other: Self) -> Self::Mask;
fn ge_mask(self, other: Self) -> Self::Mask;
fn decrease_masked(self, mask: Self::Mask) -> Self;
type UInt;
fn cast_from_int(i: Self::UInt) -> Self;
}
#[cfg(not(feature = "std"))]
#[allow(unused)]
#[allow(clippy::wrong_self_convention)]
pub(crate) trait Float: Sized {
fn is_nan(self) -> bool;
fn is_infinite(self) -> bool;
fn is_finite(self) -> bool;
}
#[allow(unused)]
pub(crate) trait FloatAsSIMD: Sized {
#[inline(always)]
fn lanes() -> usize {
1
}
#[inline(always)]
fn splat(scalar: Self) -> Self {
scalar
}
#[inline(always)]
fn extract(self, index: usize) -> Self {
debug_assert_eq!(index, 0);
self
}
#[inline(always)]
fn replace(self, index: usize, new_value: Self) -> Self {
debug_assert_eq!(index, 0);
new_value
}
}
#[allow(unused)]
pub(crate) trait BoolAsSIMD: Sized {
fn any(self) -> bool;
fn all(self) -> bool;
fn none(self) -> bool;
}
impl BoolAsSIMD for bool {
#[inline(always)]
fn any(self) -> bool {
self
}
#[inline(always)]
fn all(self) -> bool {
self
}
#[inline(always)]
fn none(self) -> bool {
!self
}
}
macro_rules! scalar_float_impl {
($ty:ident, $uty:ident) => {
#[cfg(not(feature = "std"))]
impl Float for $ty {
#[inline]
fn is_nan(self) -> bool {
self != self
}
#[inline]
fn is_infinite(self) -> bool {
self == ::core::$ty::INFINITY || self == ::core::$ty::NEG_INFINITY
}
#[inline]
fn is_finite(self) -> bool {
!(self.is_nan() || self.is_infinite())
}
}
impl FloatSIMDUtils for $ty {
type Mask = bool;
type UInt = $uty;
#[inline(always)]
fn all_lt(self, other: Self) -> bool {
self < other
}
#[inline(always)]
fn all_le(self, other: Self) -> bool {
self <= other
}
#[inline(always)]
fn all_finite(self) -> bool {
self.is_finite()
}
#[inline(always)]
fn finite_mask(self) -> Self::Mask {
self.is_finite()
}
#[inline(always)]
fn gt_mask(self, other: Self) -> Self::Mask {
self > other
}
#[inline(always)]
fn ge_mask(self, other: Self) -> Self::Mask {
self >= other
}
#[inline(always)]
fn decrease_masked(self, mask: Self::Mask) -> Self {
debug_assert!(mask, "At least one lane must be set");
<$ty>::from_bits(self.to_bits() - 1)
}
#[inline]
fn cast_from_int(i: Self::UInt) -> Self {
i as $ty
}
}
impl FloatAsSIMD for $ty {}
};
}
scalar_float_impl!(f32, u32);
scalar_float_impl!(f64, u64);