use core::{
arch::x86_64::*,
ops::{
BitAnd,
BitOr,
BitXor,
Not,
},
};
use crate::bs::word::{
Word,
gather64,
};
#[inline]
pub fn available() -> bool {
std::is_x86_feature_detected!("avx2")
}
#[derive(Copy, Clone)]
pub struct W256(pub __m256i);
impl BitAnd for W256 {
type Output = Self;
#[inline(always)]
fn bitand(self, r: Self) -> Self {
unsafe { W256(_mm256_and_si256(self.0, r.0)) }
}
}
impl BitOr for W256 {
type Output = Self;
#[inline(always)]
fn bitor(self, r: Self) -> Self {
unsafe { W256(_mm256_or_si256(self.0, r.0)) }
}
}
impl BitXor for W256 {
type Output = Self;
#[inline(always)]
fn bitxor(self, r: Self) -> Self {
unsafe { W256(_mm256_xor_si256(self.0, r.0)) }
}
}
impl Not for W256 {
type Output = Self;
#[inline(always)]
fn not(self) -> Self {
unsafe { W256(_mm256_xor_si256(self.0, _mm256_set1_epi8(-1i8))) }
}
}
impl Word for W256 {
const LANES: usize = 256;
#[inline(always)]
fn zero() -> Self {
unsafe { W256(_mm256_setzero_si256()) }
}
#[inline(always)]
fn ones() -> Self {
unsafe { W256(_mm256_set1_epi8(-1i8)) }
}
#[inline(always)]
fn gather_bitplanes(col: &[u8]) -> [W256; 8] {
let n = col.len();
let mut limbs = [[0u64; 4]; 8];
let mut l = 0;
while l * 64 < n {
let end = core::cmp::min(l * 64 + 64, n);
let sub = gather64(&col[l * 64 .. end]);
for bit in 0 .. 8 {
limbs[bit][l] = sub[bit];
}
l += 1;
}
let mut planes = [W256::zero(); 8];
for bit in 0 .. 8 {
planes[bit] =
W256(unsafe { _mm256_loadu_si256(limbs[bit].as_ptr() as *const __m256i) });
}
planes
}
#[inline(always)]
fn scatter_bitplanes(planes: &[W256; 8], col: &mut [u8]) {
if col.len() < Self::LANES {
let mut full = [0u8; 256];
Self::scatter_bitplanes(planes, &mut full);
let n = col.len();
col.copy_from_slice(&full[.. n]);
return;
}
unsafe {
let p = planes.map(|w| w.0);
let a = [
_mm256_unpacklo_epi8(p[0], p[1]),
_mm256_unpackhi_epi8(p[0], p[1]),
_mm256_unpacklo_epi8(p[2], p[3]),
_mm256_unpackhi_epi8(p[2], p[3]),
_mm256_unpacklo_epi8(p[4], p[5]),
_mm256_unpackhi_epi8(p[4], p[5]),
_mm256_unpacklo_epi8(p[6], p[7]),
_mm256_unpackhi_epi8(p[6], p[7]),
];
let mut b = [_mm256_setzero_si256(); 8];
for k in 0 .. 2 {
b[4 * k] = _mm256_unpacklo_epi16(a[k], a[2 + k]);
b[4 * k + 1] = _mm256_unpackhi_epi16(a[k], a[2 + k]);
b[4 * k + 2] = _mm256_unpacklo_epi16(a[4 + k], a[6 + k]);
b[4 * k + 3] = _mm256_unpackhi_epi16(a[4 + k], a[6 + k]);
}
let dst = col.as_mut_ptr() as *mut __m128i;
for k in 0 .. 4 {
let i = k / 2 * 4 + k % 2; let lo = _mm256_unpacklo_epi32(b[i], b[i + 2]);
let hi = _mm256_unpackhi_epi32(b[i], b[i + 2]);
for (q, x) in [(2 * k, lo), (2 * k + 1, hi)] {
let x = transpose8x4(x);
_mm_storeu_si128(dst.add(q), _mm256_castsi256_si128(x));
_mm_storeu_si128(dst.add(8 + q), _mm256_extracti128_si256::<1>(x));
}
}
}
}
}
#[inline(always)]
unsafe fn transpose8x4(mut x: __m256i) -> __m256i {
unsafe {
let mut t;
t = _mm256_and_si256(
_mm256_xor_si256(x, _mm256_srli_epi64::<7>(x)),
_mm256_set1_epi64x(0x00AA_00AA_00AA_00AA),
);
x = _mm256_xor_si256(x, _mm256_xor_si256(t, _mm256_slli_epi64::<7>(t)));
t = _mm256_and_si256(
_mm256_xor_si256(x, _mm256_srli_epi64::<14>(x)),
_mm256_set1_epi64x(0x0000_CCCC_0000_CCCC),
);
x = _mm256_xor_si256(x, _mm256_xor_si256(t, _mm256_slli_epi64::<14>(t)));
t = _mm256_and_si256(
_mm256_xor_si256(x, _mm256_srli_epi64::<28>(x)),
_mm256_set1_epi64x(0x0000_0000_F0F0_F0F0),
);
_mm256_xor_si256(x, _mm256_xor_si256(t, _mm256_slli_epi64::<28>(t)))
}
}