#[cfg(target_arch = "x86")]
use core::arch::x86 as arch_simd;
#[cfg(target_arch = "x86_64")]
use core::arch::x86_64 as arch_simd;
use core::mem::transmute;
use arch_simd::{
__m128i, _MM_HINT_T0, _mm_add_epi64, _mm_aesenc_si128, _mm_aesenclast_si128, _mm_cvtsi32_si128,
_mm_cvtsi128_si32, _mm_loadl_epi64, _mm_loadu_si128, _mm_prefetch, _mm_set1_epi32,
_mm_setzero_si128, _mm_storeu_si128, _mm_unpackhi_epi64, _mm_unpacklo_epi8, _mm_unpacklo_epi64,
_mm_xor_si128,
};
#[cfg(not(all(
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
)))]
use crate::{
arch::{ctr_simd::ctr_8x128, tier::def_hw_tier},
backend::ctr::CtrKernel,
};
use crate::{
backend::{
aes128::Aes128,
group::Lane128x8,
mode::{PostXor, PostXorAes},
},
state::State,
};
#[cfg(target_arch = "x86_64")]
mod arch_word {
use core::arch::x86_64::{
__m128i, _mm_cvtsi64_si128, _mm_cvtsi128_si64, _mm_set_epi64x, _mm_set1_epi64x,
};
#[inline(always)]
pub(super) unsafe fn to_u64(v: __m128i) -> u64 {
unsafe { _mm_cvtsi128_si64(v) as u64 }
}
#[inline(always)]
pub(super) unsafe fn seed_v(seed: u64) -> __m128i {
unsafe { _mm_set1_epi64x(seed as i64) }
}
#[inline(always)]
pub(super) unsafe fn pair_v(lo: u64, hi: u64) -> __m128i {
unsafe { _mm_set_epi64x(hi as i64, lo as i64) }
}
#[inline(always)]
pub(super) unsafe fn load_pair_u32(p0: *const u8, p1: *const u8) -> __m128i {
unsafe {
let lo = p0.cast::<u32>().read_unaligned() as u64;
let hi = p1.cast::<u32>().read_unaligned() as u64;
_mm_cvtsi64_si128((lo | (hi << 32)) as i64)
}
}
}
#[cfg(target_arch = "x86")]
mod arch_word {
use core::{
arch::x86::{__m128i, _mm_cvtsi32_si128, _mm_set_epi32, _mm_unpacklo_epi32},
mem::transmute,
};
use crate::state::State;
#[inline(always)]
pub(super) unsafe fn to_u64(v: __m128i) -> u64 {
let s: State = unsafe { transmute(v) };
u64::from_le_bytes(s.as_chunks::<8>().0[0])
}
#[inline(always)]
pub(super) unsafe fn seed_v(seed: u64) -> __m128i {
let lo = seed as i32;
let hi = (seed >> 32) as i32;
unsafe { _mm_set_epi32(hi, lo, hi, lo) }
}
#[inline(always)]
pub(super) unsafe fn pair_v(lo: u64, hi: u64) -> __m128i {
unsafe { _mm_set_epi32((hi >> 32) as i32, hi as i32, (lo >> 32) as i32, lo as i32) }
}
#[inline(always)]
pub(super) unsafe fn load_pair_u32(p0: *const u8, p1: *const u8) -> __m128i {
unsafe {
let lo = p0.cast::<i32>().read_unaligned();
let hi = p1.cast::<i32>().read_unaligned();
_mm_unpacklo_epi32(_mm_cvtsi32_si128(lo), _mm_cvtsi32_si128(hi))
}
}
}
#[derive(Clone, Copy)]
pub(crate) struct X86_128;
impl Aes128 for X86_128 {
type V = __m128i;
type Mode = PostXor;
const HAS_PREFETCH: bool = true;
#[inline(always)]
unsafe fn zero(self) -> Self::V {
unsafe { _mm_setzero_si128() }
}
#[inline(always)]
unsafe fn ld(self, ptr: *const u8) -> Self::V {
unsafe { _mm_loadu_si128(ptr.cast()) }
}
#[inline(always)]
unsafe fn st(self, ptr: *mut u8, v: Self::V) {
unsafe { _mm_storeu_si128(ptr.cast(), v) };
}
#[inline(always)]
unsafe fn to_state(self, v: Self::V) -> State {
unsafe { transmute(v) }
}
#[inline(always)]
unsafe fn to_u32(self, v: Self::V) -> u32 {
unsafe { _mm_cvtsi128_si32(v) as u32 }
}
#[inline(always)]
unsafe fn to_u64(self, v: Self::V) -> u64 {
unsafe { arch_word::to_u64(v) }
}
#[inline(always)]
unsafe fn seed_v(self, seed: u64) -> Self::V {
unsafe { arch_word::seed_v(seed) }
}
#[inline(always)]
unsafe fn pair_v(self, lo: u64, hi: u64) -> Self::V {
unsafe { arch_word::pair_v(lo, hi) }
}
#[inline(always)]
unsafe fn dup_u32(self, x: u32) -> Self::V {
unsafe { _mm_set1_epi32(x as i32) }
}
#[inline(always)]
unsafe fn xor(self, a: Self::V, b: Self::V) -> Self::V {
unsafe { _mm_xor_si128(a, b) }
}
#[inline(always)]
unsafe fn add_u64(self, a: Self::V, b: Self::V) -> Self::V {
unsafe { _mm_add_epi64(a, b) }
}
#[inline(always)]
unsafe fn prefetch_l1(self, ptr: *const u8) {
unsafe { _mm_prefetch(ptr.cast::<i8>(), _MM_HINT_T0) };
}
#[inline(always)]
unsafe fn load_pair_u64(self, p0: *const u8, p1: *const u8) -> Self::V {
unsafe { _mm_unpacklo_epi64(_mm_loadl_epi64(p0.cast()), _mm_loadl_epi64(p1.cast())) }
}
#[inline(always)]
unsafe fn load_pair_u32(self, p0: *const u8, p1: *const u8) -> Self::V {
unsafe { arch_word::load_pair_u32(p0, p1) }
}
#[inline(always)]
unsafe fn u32_v(self, x: u32) -> Self::V {
unsafe { _mm_cvtsi32_si128(x as i32) }
}
#[inline(always)]
unsafe fn zip_u8(self, v: Self::V) -> Self::V {
unsafe { _mm_unpacklo_epi8(v, _mm_unpackhi_epi64(v, v)) }
}
}
impl PostXorAes for X86_128 {
#[inline(always)]
unsafe fn post_enc(self, state: Self::V, rk: Self::V) -> Self::V {
unsafe { _mm_aesenc_si128(state, rk) }
}
#[inline(always)]
unsafe fn post_enclast(self, state: Self::V, rk: Self::V) -> Self::V {
unsafe { _mm_aesenclast_si128(state, rk) }
}
}
impl Lane128x8 for X86_128 {}
#[cfg(not(all(
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
)))]
impl CtrKernel for X86_128 {
#[inline(always)]
unsafe fn ctr<const XOR: bool>(self, root: &[u8; 32], blk: u64, buf: &mut [u8]) {
unsafe { ctr_8x128::<X86_128, XOR>(X86_128, root, blk, buf) };
}
}
#[cfg(not(all(
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
)))]
macro_rules! def_128_tier {
($mod_name:ident, $feat_cfg:meta, $feat:literal) => {
pub(super) mod $mod_name {
use super::*;
def_hw_tier!(
X86_128,
X86_128,
__m128i,
$feat_cfg,
any(),
all(target_feature = "aes", target_feature = "sse2"),
$feat
);
}
};
}
#[cfg(all(
not(all(
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
)),
not(all(
target_feature = "aes",
target_feature = "avx",
target_feature = "sse2"
)),
any(feature = "std", all(target_feature = "aes", target_feature = "sse2"))
))]
def_128_tier!(
sse2_aes,
all(target_feature = "aes", target_feature = "sse2"),
"aes,sse2"
);
#[cfg(all(
not(all(
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
)),
any(
feature = "std",
all(
target_feature = "aes",
target_feature = "avx",
target_feature = "sse2"
)
)
))]
def_128_tier!(
avx_aes,
all(
target_feature = "aes",
target_feature = "avx",
target_feature = "sse2"
),
"aes,avx,sse2"
);