#[cfg(target_arch = "x86")]
use core::arch::x86 as arch_simd;
#[cfg(target_arch = "x86_64")]
use core::arch::x86_64 as arch_simd;
use arch_simd::{
__m128i, __m256i, _mm256_add_epi64, _mm256_aesenc_epi128, _mm256_aesenclast_epi128,
_mm256_castsi256_si128, _mm256_extracti128_si256, _mm256_loadu_si256, _mm256_setzero_si256,
_mm256_storeu_si256, _mm256_xor_si256, _mm256_zextsi128_si256,
};
use super::aes128::X86_128;
use crate::{
arch::wide::{Lane256x4, WideAes, ctr_4_wide},
backend::ctr::ctr_1x128,
};
#[cfg(not(all(
target_arch = "x86_64",
target_feature = "avx512f",
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
)))]
use crate::{
arch::{tier::def_hw_tier, wide::Wide256},
backend::ctr::{CtrKernel, split_root},
};
#[cfg(target_arch = "x86_64")]
mod arch_bcast {
use core::arch::x86_64::{__m128i, __m256i, _mm256_broadcastsi128_si256};
#[inline(always)]
pub(super) unsafe fn broadcast_128(v: __m128i) -> __m256i {
unsafe { _mm256_broadcastsi128_si256(v) }
}
}
#[cfg(target_arch = "x86")]
mod arch_bcast {
use core::arch::x86::{__m128i, __m256i, _mm256_set_m128i};
#[inline(always)]
pub(super) unsafe fn broadcast_128(v: __m128i) -> __m256i {
unsafe { _mm256_set_m128i(v, v) }
}
}
#[derive(Clone, Copy)]
pub(crate) struct Vaes256;
impl WideAes for Vaes256 {
type L = X86_128;
type W = __m256i;
const BLOCK_PER_VEC: usize = 2;
#[inline(always)]
fn lane(self) -> X86_128 {
X86_128
}
#[inline(always)]
unsafe fn zero_w(self) -> Self::W {
unsafe { _mm256_setzero_si256() }
}
#[inline(always)]
unsafe fn ld_w(self, ptr: *const u8) -> Self::W {
unsafe { _mm256_loadu_si256(ptr.cast()) }
}
#[inline(always)]
unsafe fn st_w(self, ptr: *mut u8, v: Self::W) {
unsafe { _mm256_storeu_si256(ptr.cast(), v) };
}
#[inline(always)]
unsafe fn zext128(self, v: __m128i) -> Self::W {
unsafe { _mm256_zextsi128_si256(v) }
}
#[inline(always)]
unsafe fn broadcast(self, v: __m128i) -> Self::W {
unsafe { arch_bcast::broadcast_128(v) }
}
#[inline(always)]
unsafe fn xor_w(self, a: Self::W, b: Self::W) -> Self::W {
unsafe { _mm256_xor_si256(a, b) }
}
#[inline(always)]
unsafe fn add_u64_w(self, a: Self::W, b: Self::W) -> Self::W {
unsafe { _mm256_add_epi64(a, b) }
}
#[inline(always)]
unsafe fn aesenc_w(self, a: Self::W, rk: Self::W) -> Self::W {
unsafe { _mm256_aesenc_epi128(a, rk) }
}
#[inline(always)]
unsafe fn aesenclast_w(self, a: Self::W, rk: Self::W) -> Self::W {
unsafe { _mm256_aesenclast_epi128(a, rk) }
}
}
impl Lane256x4 for Vaes256 {
#[inline(always)]
unsafe fn split_pair(self, v: __m256i) -> (__m128i, __m128i) {
unsafe { (_mm256_castsi256_si128(v), _mm256_extracti128_si256::<1>(v)) }
}
}
#[inline(always)]
pub(super) unsafe fn ctr_with_rk<const XOR: bool>(
rk_li: [__m128i; 4],
mut blk: u64,
mut ptr: *mut u8,
mut rem: usize,
) {
unsafe {
ctr_4_wide::<Vaes256, XOR>(Vaes256, rk_li, &mut blk, &mut ptr, &mut rem);
ctr_1x128::<X86_128, XOR>(X86_128, rk_li, blk, ptr, rem);
}
}
#[cfg(not(all(
target_arch = "x86_64",
target_feature = "avx512f",
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
)))]
impl CtrKernel for Vaes256 {
#[inline(always)]
unsafe fn ctr<const XOR: bool>(self, root: &[u8; 32], blk: u64, buf: &mut [u8]) {
unsafe {
let rk_li = split_root(self.lane(), root);
ctr_with_rk::<XOR>(rk_li, blk, buf.as_mut_ptr(), buf.len());
}
}
}
#[cfg(not(all(
target_arch = "x86_64",
target_feature = "avx512f",
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
)))]
def_hw_tier!(
Wide256(Vaes256),
Vaes256,
__m128i,
all(
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
),
all(
target_arch = "x86",
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
),
all(target_feature = "aes", target_feature = "sse2"),
"vaes,avx2,aes"
);