pub(crate) mod i32 {
pub(crate) type Vector = __m256i;
use std::arch::x86_64::*;
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) unsafe fn load(p: *const i32) -> __m256i {
unsafe { _mm256_loadu_si256(p.cast()) }
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) unsafe fn store(p: *mut i32, v: __m256i) {
unsafe { _mm256_storeu_si256(p.cast(), v) };
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn shr<const N: i32>(v: __m256i) -> __m256i {
_mm256_srai_epi32::<N>(v)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn select(m: __m256i, a: __m256i, b: __m256i) -> __m256i {
_mm256_blendv_epi8(b, a, m)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn splat(v: i32) -> __m256i {
_mm256_set1_epi32(v)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn add(a: __m256i, b: __m256i) -> __m256i {
_mm256_add_epi32(a, b)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn sub(a: __m256i, b: __m256i) -> __m256i {
_mm256_sub_epi32(a, b)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn and(a: __m256i, b: __m256i) -> __m256i {
_mm256_and_si256(a, b)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn or(a: __m256i, b: __m256i) -> __m256i {
_mm256_or_si256(a, b)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn gt(a: __m256i, b: __m256i) -> __m256i {
_mm256_cmpgt_epi32(a, b)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn abs(v: __m256i) -> __m256i {
_mm256_abs_epi32(v)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) unsafe fn load_u8(p: *const u8) -> __m256i {
_mm256_cvtepu8_epi32(unsafe { _mm_loadl_epi64(p.cast()) })
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) unsafe fn load_u16(p: *const u16) -> __m256i {
_mm256_cvtepu16_epi32(unsafe { _mm_loadu_si128(p.cast()) })
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn shl<const N: i32>(v: __m256i) -> __m256i {
_mm256_slli_epi32::<N>(v)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn xor(a: __m256i, b: __m256i) -> __m256i {
_mm256_xor_si256(a, b)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn min(a: __m256i, b: __m256i) -> __m256i {
_mm256_min_epi32(a, b)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn max(a: __m256i, b: __m256i) -> __m256i {
_mm256_max_epi32(a, b)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn lt_unsigned(a: __m256i, b: __m256i) -> __m256i {
let sign = _mm256_set1_epi32(i32::MIN);
_mm256_cmpgt_epi32(xor(b, sign), xor(a, sign))
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn shr_unsigned<const N: i32>(v: __m256i) -> __m256i {
_mm256_srli_epi32::<N>(v)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn to_f32_bits(v: __m256i) -> __m256i {
_mm256_castps_si256(_mm256_cvtepi32_ps(v))
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) unsafe fn load_pairs(p: *const i32) -> (__m256i, __m256i) {
let a = _mm256_castsi256_ps(unsafe { load(p) });
let b = _mm256_castsi256_ps(unsafe { load(p.add(8)) });
let order = _mm256_setr_epi32(0, 1, 4, 5, 2, 3, 6, 7);
(
_mm256_permutevar8x32_epi32(
_mm256_castps_si256(_mm256_shuffle_ps::<0x88>(a, b)),
order,
),
_mm256_permutevar8x32_epi32(
_mm256_castps_si256(_mm256_shuffle_ps::<0xdd>(a, b)),
order,
),
)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn mulhi(a: __m256i, b: __m256i) -> __m256i {
let even = _mm256_srli_epi64::<32>(_mm256_mul_epi32(a, b));
let odd = _mm256_mul_epi32(_mm256_srli_epi64::<32>(a), _mm256_srli_epi64::<32>(b));
_mm256_blend_epi32::<0xaa>(even, odd)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn eq(a: __m256i, b: __m256i) -> __m256i {
_mm256_cmpeq_epi32(a, b)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn bits_clear(v: __m256i, mask: __m256i) -> bool {
_mm256_testz_si256(v, mask) != 0
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn mul(a: __m256i, b: __m256i) -> __m256i {
_mm256_mullo_epi32(a, b)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn shr_by(v: __m256i, count: u32) -> __m256i {
_mm256_sra_epi32(v, _mm_cvtsi32_si128(count as i32))
}
}
pub(crate) mod f32 {
use std::arch::x86_64::*;
pub(crate) use std::arch::x86_64::{
_mm256_add_ps as add, _mm256_div_ps as div, _mm256_loadu_ps as load, _mm256_max_ps as max,
_mm256_min_ps as min, _mm256_mul_ps as mul, _mm256_set1_ps as splat,
_mm256_storeu_ps as store, _mm256_sub_ps as sub,
};
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn abs(v: __m256) -> __m256 {
_mm256_andnot_ps(_mm256_set1_ps(-0.0), v)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn mask_nonzero(v: __m256, a: __m256, b: __m256) -> __m256 {
_mm256_and_ps(
v,
_mm256_and_ps(
_mm256_cmp_ps::<_CMP_NEQ_UQ>(a, _mm256_setzero_ps()),
_mm256_cmp_ps::<_CMP_NEQ_UQ>(b, _mm256_setzero_ps()),
),
)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn from_i32(v: __m256i) -> __m256 {
_mm256_cvtepi32_ps(v)
}
#[inline]
#[target_feature(enable = "avx2")]
pub(crate) fn trunc_to_i32(v: __m256) -> __m256i {
_mm256_cvttps_epi32(v)
}
}