use generic_array::{
GenericArray,
sequence::GenericSequence,
typenum::{self, Unsigned},
};
use crate::{
isa::InstructionSet,
register::{
BitshiftRegister, BitwiseRegister, CastRegister, CoreRegister, ExtendRegister, IntegerRegister,
InterleaveRegister, MaskElement, MaskRegister, NumericRegister, PartialOrdRegister, PermuteRegister, Register,
ShuffleRegister, Storage, UnsignedIntegerRegister, array::ArrayRegister, empty_reg, reg,
},
};
use super::arch;
#[cfg_attr(not(feature = "document_registers"), doc(hidden))]
#[derive(Debug, Clone, Copy, Hash)]
pub struct U32x4V2;
#[thermite_macros::inline_always]
impl CoreRegister for U32x4V2 {
type NativeIsa = crate::backend::x86_v2::X86V2;
type Lanes = typenum::U4;
type Storage = arch::__m128i;
type Mask = Self;
const IS_EMULATED: bool = false;
const EMPTY: Storage<Self> = empty_reg::<Self>();
const HAS_EQUAL_SIZE_MASK: bool = true;
fn blendv(mask: Storage<Self::Mask>, lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_blendv_epi8(lhs, rhs, mask) }
}
fn zz(mask: Storage<Self::Mask>, value: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_and_si128(value, mask) }
}
fn nz(mask: Storage<Self::Mask>, value: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_andnot_si128(mask, value) }
}
fn zeroupper_z<Z: crate::register::ZeroUpper>(value: Storage<Self>) -> Storage<Self> {
super::I32x4V2::zeroupper_z::<Z>(value) }
fn from_mask(mask: Storage<Self::Mask>) -> Storage<Self> {
mask
}
}
#[thermite_macros::inline_always]
impl MaskRegister for U32x4V2 {
fn set(mut mask: Storage<Self::Mask>, lane: usize, value: bool) -> Storage<Self> {
Self::as_mut_slice(&mut mask)[lane] = if value { MaskElement::TRUTHY } else { MaskElement::FALSY };
mask
}
fn test(mask: Storage<Self::Mask>, lane: usize) -> bool {
Self::as_slice(&mask)[lane].to_bool()
}
const FALSY: Storage<Self> = reg::<Self, 4>([0; 4]);
const TRUTHY: Storage<Self> = reg::<Self, 4>([!0; 4]);
fn new_mask(value: GenericArray<bool, Self::Lanes>) -> Storage<Self> {
unsafe { arch::_mm_cvtboolx4_to_epi32_mask_v2(value) }
}
fn all(value: Storage<Self>) -> bool {
unsafe { arch::_mm_movemask_epi8(value) as u32 == 0xFFFF }
}
fn any(value: Storage<Self>) -> bool {
unsafe { arch::_mm_movemask_epi8(value) != 0 }
}
fn none(value: Storage<Self>) -> bool {
unsafe { arch::_mm_movemask_epi8(value) == 0 }
}
fn count_set<const N: usize>(values: [Storage<Self>; N]) -> usize {
unsafe { arch::_mm_count_mask_epi32x_v1(values) }
}
fn from_native_bitmask(bitmask: u64) -> Storage<Self> {
unsafe { arch::_mm_movm_epi32x_v1(bitmask) }
}
fn native_bitmask(value: Storage<Self>) -> Option<u64> {
Some(unsafe { arch::_mm_movemask_ps(arch::_mm_castsi128_ps(value)) as u64 })
}
#[cfg(feature = "bitvec")]
fn fill_bitmask(value: Storage<Self>, view: &mut bitvec::slice::BitSlice<u32>) {
let mask = unsafe { arch::_mm_movemask_ps(arch::_mm_castsi128_ps(value)) as u32 };
let mask = bitvec::slice::BitSlice::from_slice(core::slice::from_ref(&mask));
view.copy_from_bitslice(&mask[..Self::Lanes::USIZE]);
}
}
#[rustfmt::skip] #[thermite_macros::inline_always]
impl BitwiseRegister for U32x4V2 {
fn bitxor(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_xor_si128(lhs, rhs) }
}
fn bitand(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_and_si128(lhs, rhs) }
}
fn bitandnot(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_andnot_si128(lhs, rhs) }
}
fn bitor(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_or_si128(lhs, rhs) }
}
fn not(value: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_xor_si128(value, arch::_mm_set1_epi8(-1)) }
}
}
#[thermite_macros::inline_always]
impl ExtendRegister<u32> for U32x4V2 {
fn extend(value: Storage<u32>) -> Storage<Self> {
unsafe { arch::_mm_setr_epu32x(value, 0, 0, 0) }
}
fn narrow(value: Storage<Self>) -> Storage<u32> {
unsafe { arch::_mm_cvtsi128_si32(value) as u32 }
}
}
#[thermite_macros::inline_always]
impl InterleaveRegister for U32x4V2 {
fn interleave(a: Storage<Self>, b: Storage<Self>) -> (Storage<Self>, Storage<Self>) {
super::I32x4V2::interleave(a, b)
}
fn deinterleave(a: Storage<Self>, b: Storage<Self>) -> (Storage<Self>, Storage<Self>) {
super::I32x4V2::deinterleave(a, b)
}
}
#[thermite_macros::inline_always]
impl Register for U32x4V2 {
type Element = u32;
type Signed = super::I32x4V2;
type Unsigned = super::U32x4V2;
fn into_mask(value: Storage<Self>) -> Storage<Self::Mask> {
Self::ne(value, Self::ZERO)
}
fn into_mask_unchecked(value: Storage<Self>) -> Storage<Self::Mask> {
value
}
fn msb_to_mask(value: Storage<Self>) -> Storage<Self::Mask> {
unsafe { arch::_mm_srai_epi32(value, 31) }
}
fn new(value: GenericArray<Self::Element, Self::Lanes>) -> Storage<Self> {
unsafe { arch::_mm_loadu_si128(value.as_ptr() as *const _) }
}
fn single(value: Self::Element) -> Storage<Self> {
unsafe { arch::_mm_setr_epi32(value as i32, 0, 0, 0) }
}
impl_native_radix3!(arch::_mm_interleave3_epi32, arch::_mm_deinterleave3_epi32);
impl_native_extract!(@epi32x4);
fn splat(value: Self::Element) -> Storage<Self> {
unsafe { arch::_mm_set1_epi32(value as i32) }
}
unsafe fn load(ptr: *const Self::Element) -> Storage<Self> {
unsafe { arch::_mm_load_si128(ptr as *const _) }
}
unsafe fn load_unaligned(ptr: *const Self::Element) -> Storage<Self> {
unsafe { arch::_mm_loadu_si128(ptr as *const _) }
}
unsafe fn store(ptr: *mut Self::Element, value: Storage<Self>) {
unsafe { arch::_mm_store_si128(ptr as *mut _, value) }
}
unsafe fn store_unaligned(ptr: *mut Self::Element, value: Storage<Self>) {
unsafe { arch::_mm_storeu_si128(ptr as *mut _, value) }
}
unsafe fn load_stream(ptr: *const Self::Element) -> Storage<Self> {
unsafe { arch::_mm_stream_load_si128(ptr as _) }
}
unsafe fn store_stream(ptr: *mut Self::Element, value: Storage<Self>) {
unsafe { arch::_mm_stream_si128(ptr as _, value) }
}
fn reverse(value: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_shuffle_epi32::<{ MM_SHUFFLE!(0, 1, 2, 3) }>(value) }
}
fn swap_bytes(value: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_bswap_epi32x_v2(value) }
}
const HAS_PERMUTEV: bool = true;
impl_byte_align_alignr!();
fn permutev(value: Storage<Self>, idxs: GenericArray<u32, Self::Lanes>) -> Storage<Self> {
unsafe { arch::_mm_permutevarx_epi32x_v2(value, core::mem::transmute(idxs)) }
}
compress_via_table!();
}
#[thermite_macros::inline_always]
impl ShuffleRegister for U32x4V2 {
fn shuffle<const IMM8: i32>(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
unsafe {
arch::_mm_castps_si128(arch::_mm_shuffle_ps(
arch::_mm_castsi128_ps(lhs),
arch::_mm_castsi128_ps(rhs),
IMM8,
))
}
}
}
#[thermite_macros::inline_always]
impl PermuteRegister for U32x4V2 {
fn permute<const IMM8: i32>(value: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_shuffle_epi32(value, IMM8) }
}
}
#[rustfmt::skip] #[thermite_macros::inline_always]
impl BitshiftRegister for U32x4V2 {
const HAS_TRUE_SHIFTV: bool = false;
const HAS_WIDE_BYTE_SHIFTS: bool = true;
fn bshli<const IMM8: i32>(value: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_bslli_si128(value, IMM8) }
}
fn bshri<const IMM8: i32>(value: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_bsrli_si128(value, IMM8) }
}
fn shl(value: Storage<Self>, shift: u32) -> Storage<Self> {
unsafe { arch::_mm_sll_epi32(value, arch::_mm_cvtsi32_si128(shift as i32)) }
}
fn shr(value: Storage<Self>, shift: u32) -> Storage<Self> {
unsafe { arch::_mm_srl_epi32(value, arch::_mm_cvtsi32_si128(shift as i32)) }
}
fn shlv(value: Storage<Self>, shifts: Storage<Self::Unsigned>) -> Storage<Self> {
unsafe { arch::_mm_sllv_epi32x_v1(value, shifts) }
}
fn shrv(value: Storage<Self>, shifts: Storage<Self::Unsigned>) -> Storage<Self> {
unsafe { arch::_mm_srlv_epi32x_v1(value, shifts) }
}
fn shli<const IMM8: i32>(value: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_slli_epi32(value, IMM8) }
}
fn shri<const IMM8: i32>(value: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_srli_epi32(value, IMM8) }
}
}
#[thermite_macros::inline_always]
impl PartialOrdRegister for U32x4V2 {
fn lt(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_cmplt_epu32x_v2(lhs, rhs) }
}
fn le(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_cmple_epu32x_v2(lhs, rhs) }
}
fn gt(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_cmpgt_epu32x_v2(lhs, rhs) }
}
fn ge(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_cmpge_epu32x_v2(lhs, rhs) }
}
fn eq(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_cmpeq_epi32(lhs, rhs) }
}
}
#[thermite_macros::inline_always]
impl NumericRegister for U32x4V2 {
sort_via_network!(4);
const ZERO: Storage<Self> = reg::<Self, 4>([0; 4]);
const ONE: Storage<Self> = reg::<Self, 4>([1; 4]);
const TWO: Storage<Self> = reg::<Self, 4>([2; 4]);
const MIN: Storage<Self> = reg::<Self, 4>([u32::MIN; 4]);
const MAX: Storage<Self> = reg::<Self, 4>([u32::MAX; 4]);
fn min_element(value: Storage<Self>) -> Self::Element {
_mm_reduce_epi32_v1!(value; _mm_min_epu32 _mm_min_epu32) as u32
}
fn max_element(value: Storage<Self>) -> Self::Element {
_mm_reduce_epi32_v1!(value; _mm_max_epu32 _mm_max_epu32) as u32
}
fn sum_elements(value: Storage<Self>) -> Self::Element {
_mm_reduce_epi32_v1!(value; _mm_add_epi32 _mm_add_epi32) as u32
}
fn prod_elements(value: Storage<Self>) -> Self::Element {
_mm_reduce_epi32_v1!(value; _mm_mullo_epi32 _mm_mullo_epi32) as u32
}
fn pairwise_sum(lo: Storage<Self>, hi: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_hadd_epi32(lo, hi) }
}
fn offset() -> Storage<Self> {
Self::splat(<Self::Lanes as typenum::Unsigned>::U32)
}
fn indexed() -> Storage<Self> {
Self::new(GenericArray::generate(|i| i as u32))
}
fn add(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_add_epi32(lhs, rhs) }
}
fn sub(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_sub_epi32(lhs, rhs) }
}
fn add_c(mask: Storage<Self::Mask>, lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_add_epi32(lhs, arch::_mm_and_si128(rhs, mask)) }
}
fn sub_c(mask: Storage<Self::Mask>, lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_sub_epi32(lhs, arch::_mm_and_si128(rhs, mask)) }
}
fn mul(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_mullo_epi32(lhs, rhs) }
}
fn div(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
Self::zip(lhs, rhs, |a, b| if b == 0 { 0 } else { a / b })
}
fn rem(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
Self::zip(lhs, rhs, |a, b| if b == 0 { 0 } else { a % b })
}
fn min(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_min_epu32(lhs, rhs) }
}
fn max(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_max_epu32(lhs, rhs) }
}
}
#[thermite_macros::inline_always]
impl IntegerRegister for U32x4V2 {
fn mulhi(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_mullhi_epu32x_v1(lhs, rhs) }
}
fn mullo(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_mullo_epi32(lhs, rhs) }
}
fn saturating_add(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
Self::add(rhs, Self::min(lhs, Self::not(rhs)))
}
fn saturating_sub(lhs: Storage<Self>, rhs: Storage<Self>) -> Storage<Self> {
Self::sub(Self::max(lhs, rhs), rhs)
}
fn wrapping_sum(value: Storage<Self>) -> Self::Element {
_mm_reduce_epi32_v1!(value; _mm_add_epi32 _mm_add_epi32) as u32
}
fn wrapping_product(value: Storage<Self>) -> Self::Element {
_mm_reduce_epi32_v1!(value; _mm_mullo_epi32 _mm_mullo_epi32) as u32
}
fn div_branched(value: Storage<Self>, divider: crate::Divider<Self::Element>) -> Storage<Self> {
arch::div_epu::<Self>(value, divider.multiplier(), divider.shift())
}
fn div_branchfree(value: Storage<Self>, divider: crate::BranchfreeDivider<Self::Element>) -> Storage<Self> {
arch::div_epu_bf::<Self>(value, divider.multiplier(), divider.shift())
}
fn divv_branchfree(value: Storage<Self>, dividers: crate::divider::vector::VectorDivider<Self>) -> Storage<Self> {
arch::divv_epu_bf::<Self>(value, dividers.multipliers.0, dividers.shifts.0)
}
const HAS_HARDWARE_POPCNT: bool = false;
fn count_ones(value: Storage<Self>) -> Storage<Self> {
unsafe { arch::_mm_popcnt_epi32x_v2(value) }
}
fn leading_zeros(value: Storage<Self>) -> Storage<Self> {
Self::sub(Self::splat(32), Self::ilog2p1(value))
}
fn trailing_zeros(value: Storage<Self>) -> Storage<Self> {
super::I32x4V2::trailing_zeros(value)
}
}
#[thermite_macros::inline_always]
impl UnsignedIntegerRegister for U32x4V2 {
fn morton<const N: usize>(values: [Storage<Self>; N]) -> Storage<Self> {
if const { N == 2 } {
unsafe { arch::_mm_morton2_epu32x_v2(values[0], values[1]) }
} else {
crate::backend::generic::polyfills::morton_cascade::<Self, N>(values)
}
}
fn reverse_morton<const N: usize>(code: Storage<Self>) -> [Storage<Self>; N] {
if const { N == 2 } {
unsafe {
crate::backend::generic::polyfills::morton_pack2::<Self, N>(
arch::_mm_morton2_compress_epu32x_v2(code),
arch::_mm_morton2_compress_epu32x_v2(arch::_mm_srli_epi32(code, 1)),
)
}
} else {
crate::backend::generic::polyfills::reverse_morton_cascade::<Self, N>(code)
}
}
}
#[thermite_macros::inline_always]
impl CastRegister<U32x4V2> for ArrayRegister<super::U64x2V2, 2> {
fn cast_from(value: Storage<U32x4V2>) -> Storage<Self> {
unsafe {
let lo = arch::_mm_cvtepu32_epi64(value);
let hi = arch::_mm_cvtepu32_epi64(arch::_mm_srli_si128(value, 8));
ArrayRegister([lo, hi])
}
}
}