#![cfg(target_arch = "x86_64")]
#![allow(unsafe_code)]
use std::arch::x86_64::{
__m256d, _mm256_add_pd, _mm256_andnot_pd, _mm256_blend_pd, _mm256_blendv_pd, _mm256_cmp_pd,
_mm256_div_pd, _mm256_extractf128_pd, _mm256_fmadd_pd, _mm256_loadu_pd, _mm256_max_pd,
_mm256_min_pd, _mm256_mul_pd, _mm256_permute2f128_pd, _mm256_permute4x64_pd, _mm256_set1_pd,
_mm256_setzero_pd, _mm256_sqrt_pd, _mm256_storeu_pd, _mm256_sub_pd, _mm256_unpackhi_pd,
_mm256_unpacklo_pd, _mm_cvtsd_f64, _mm_unpackhi_pd, _CMP_GT_OQ,
};
use crate::{Kernel, Simd};
#[derive(Debug, Clone, Copy)]
pub struct Avx2(());
impl Avx2 {
#[cfg(wickra_avx512)]
pub(crate) const fn implied() -> Self {
Self(())
}
}
#[inline]
pub(crate) fn run<K: Kernel>(kernel: K) -> K::Output {
debug_assert!(matches!(
crate::level(),
crate::Level::Avx2Fma | crate::Level::Avx512
));
unsafe { run_avx2_fma(kernel) }
}
#[target_feature(enable = "avx2,fma")]
unsafe fn run_avx2_fma<K: Kernel>(kernel: K) -> K::Output {
kernel.run(Avx2(()))
}
#[allow(clippy::inline_always)]
impl Simd for Avx2 {
type V = __m256d;
#[inline(always)]
fn splat(self, x: f64) -> __m256d {
unsafe { _mm256_set1_pd(x) }
}
#[inline(always)]
fn load(self, a: &[f64; 4]) -> __m256d {
unsafe { _mm256_loadu_pd(a.as_ptr()) }
}
#[inline(always)]
fn store(self, v: __m256d, a: &mut [f64; 4]) {
unsafe { _mm256_storeu_pd(a.as_mut_ptr(), v) }
}
#[inline(always)]
fn to_array(self, v: __m256d) -> [f64; 4] {
let mut a = [0.0; 4];
self.store(v, &mut a);
a
}
#[inline(always)]
fn add(self, a: __m256d, b: __m256d) -> __m256d {
unsafe { _mm256_add_pd(a, b) }
}
#[inline(always)]
fn sub(self, a: __m256d, b: __m256d) -> __m256d {
unsafe { _mm256_sub_pd(a, b) }
}
#[inline(always)]
fn mul(self, a: __m256d, b: __m256d) -> __m256d {
unsafe { _mm256_mul_pd(a, b) }
}
#[inline(always)]
fn div(self, a: __m256d, b: __m256d) -> __m256d {
unsafe { _mm256_div_pd(a, b) }
}
#[inline(always)]
fn sqrt(self, a: __m256d) -> __m256d {
unsafe { _mm256_sqrt_pd(a) }
}
#[inline(always)]
fn mul_add(self, a: __m256d, b: __m256d, c: __m256d) -> __m256d {
unsafe { _mm256_fmadd_pd(a, b, c) }
}
#[inline(always)]
fn max(self, a: __m256d, b: __m256d) -> __m256d {
unsafe { _mm256_max_pd(a, b) }
}
#[inline(always)]
fn min(self, a: __m256d, b: __m256d) -> __m256d {
unsafe { _mm256_min_pd(a, b) }
}
#[inline(always)]
fn abs(self, a: __m256d) -> __m256d {
unsafe { _mm256_andnot_pd(_mm256_set1_pd(-0.0), a) }
}
#[inline(always)]
fn select_positive(self, test: __m256d, yes: __m256d, no: __m256d) -> __m256d {
unsafe {
let positive = _mm256_cmp_pd::<_CMP_GT_OQ>(test, _mm256_setzero_pd());
_mm256_blendv_pd(no, yes, positive)
}
}
#[inline(always)]
fn shift1(self, v: __m256d) -> __m256d {
unsafe {
_mm256_blend_pd::<0b0001>(
_mm256_permute4x64_pd::<0b10_01_00_00>(v),
_mm256_setzero_pd(),
)
}
}
#[inline(always)]
fn shift2(self, v: __m256d) -> __m256d {
unsafe {
_mm256_blend_pd::<0b0011>(
_mm256_permute4x64_pd::<0b01_00_00_00>(v),
_mm256_setzero_pd(),
)
}
}
#[inline(always)]
fn broadcast_last(self, v: __m256d) -> __m256d {
unsafe { _mm256_permute4x64_pd::<0b11_11_11_11>(v) }
}
#[inline(always)]
fn last_lane(self, v: __m256d) -> f64 {
unsafe {
let hi = _mm256_extractf128_pd::<1>(v);
_mm_cvtsd_f64(_mm_unpackhi_pd(hi, hi))
}
}
#[inline(always)]
fn transpose4(self, a: __m256d, b: __m256d, c: __m256d, d: __m256d) -> [__m256d; 4] {
unsafe {
let ab_even = _mm256_unpacklo_pd(a, b);
let ab_odd = _mm256_unpackhi_pd(a, b);
let cd_even = _mm256_unpacklo_pd(c, d);
let cd_odd = _mm256_unpackhi_pd(c, d);
[
_mm256_permute2f128_pd::<0x20>(ab_even, cd_even),
_mm256_permute2f128_pd::<0x20>(ab_odd, cd_odd),
_mm256_permute2f128_pd::<0x31>(ab_even, cd_even),
_mm256_permute2f128_pd::<0x31>(ab_odd, cd_odd),
]
}
}
type W = [__m256d; 2];
#[inline(always)]
fn load8(self, a: &[f64; 8]) -> [__m256d; 2] {
let (lo, hi) = a.split_at(4);
[
self.load(lo.try_into().expect("four lanes")),
self.load(hi.try_into().expect("four lanes")),
]
}
#[inline(always)]
fn store8(self, v: [__m256d; 2], a: &mut [f64; 8]) {
let (lo, hi) = a.split_at_mut(4);
self.store(v[0], lo.try_into().expect("four lanes"));
self.store(v[1], hi.try_into().expect("four lanes"));
}
#[inline(always)]
fn splat8(self, x: f64) -> [__m256d; 2] {
[self.splat(x); 2]
}
#[inline(always)]
fn add8(self, a: [__m256d; 2], b: [__m256d; 2]) -> [__m256d; 2] {
[self.add(a[0], b[0]), self.add(a[1], b[1])]
}
#[inline(always)]
fn sub8(self, a: [__m256d; 2], b: [__m256d; 2]) -> [__m256d; 2] {
[self.sub(a[0], b[0]), self.sub(a[1], b[1])]
}
#[inline(always)]
fn mul8(self, a: [__m256d; 2], b: [__m256d; 2]) -> [__m256d; 2] {
[self.mul(a[0], b[0]), self.mul(a[1], b[1])]
}
#[inline(always)]
fn div8(self, a: [__m256d; 2], b: [__m256d; 2]) -> [__m256d; 2] {
[self.div(a[0], b[0]), self.div(a[1], b[1])]
}
#[inline(always)]
fn sqrt8(self, a: [__m256d; 2]) -> [__m256d; 2] {
[self.sqrt(a[0]), self.sqrt(a[1])]
}
#[inline(always)]
fn max8(self, a: [__m256d; 2], b: [__m256d; 2]) -> [__m256d; 2] {
[self.max(a[0], b[0]), self.max(a[1], b[1])]
}
#[inline(always)]
fn select_positive8(
self,
test: [__m256d; 2],
yes: [__m256d; 2],
no: [__m256d; 2],
) -> [__m256d; 2] {
[
self.select_positive(test[0], yes[0], no[0]),
self.select_positive(test[1], yes[1], no[1]),
]
}
}