#[cfg(any(feature = "std", all(target_feature = "aes", target_feature = "neon")))]
mod hw {
use core::{
arch::{
aarch64::{
uint8x16_t, vaddq_u64, vcombine_u64, vcreate_u64, vdupq_n_u8, vdupq_n_u32, vdupq_n_u64,
veorq_u8, vextq_u8, vgetq_lane_u32, vgetq_lane_u64, vld1_u64, vld1q_u8,
vreinterpretq_u8_u32, vreinterpretq_u8_u64, vreinterpretq_u32_u8, vreinterpretq_u64_u8,
vst1q_u8, vzip1q_u8,
},
asm,
},
mem::transmute,
};
use crate::{
arch::{ctr_simd::ctr_8x128, tier::def_hw_tier},
backend::{
aes128::Aes128,
ctr::CtrKernel,
group::Lane128x8,
mode::{PreXor, PreXorAes},
},
state::State,
};
#[inline(always)]
unsafe fn fmov_u64(x: u64) -> uint8x16_t {
let v: uint8x16_t;
unsafe {
asm!(
"fmov {v:d}, {x}",
v = out(vreg) v,
x = in(reg) x,
options(pure, nomem, nostack, preserves_flags),
);
}
v
}
#[derive(Clone, Copy)]
pub(super) struct Neon128;
impl Aes128 for Neon128 {
type V = uint8x16_t;
type Mode = PreXor;
#[inline(always)]
unsafe fn zero(self) -> Self::V {
unsafe { vdupq_n_u8(0) }
}
#[inline(always)]
unsafe fn ld(self, ptr: *const u8) -> Self::V {
unsafe { vld1q_u8(ptr) }
}
#[inline(always)]
unsafe fn st(self, ptr: *mut u8, v: Self::V) {
unsafe { vst1q_u8(ptr, v) };
}
#[inline(always)]
unsafe fn to_state(self, v: Self::V) -> State {
unsafe { transmute(v) }
}
#[inline(always)]
unsafe fn to_u32(self, v: Self::V) -> u32 {
unsafe { vgetq_lane_u32::<0>(vreinterpretq_u32_u8(v)) }
}
#[inline(always)]
unsafe fn to_u64(self, v: Self::V) -> u64 {
unsafe { vgetq_lane_u64::<0>(vreinterpretq_u64_u8(v)) }
}
#[inline(always)]
unsafe fn seed_v(self, seed: u64) -> Self::V {
unsafe { vreinterpretq_u8_u64(vdupq_n_u64(seed)) }
}
#[inline(always)]
unsafe fn pair_v(self, lo: u64, hi: u64) -> Self::V {
unsafe { vreinterpretq_u8_u64(vcombine_u64(vcreate_u64(lo), vcreate_u64(hi))) }
}
#[inline(always)]
unsafe fn dup_u32(self, x: u32) -> Self::V {
unsafe { vreinterpretq_u8_u32(vdupq_n_u32(x)) }
}
#[inline(always)]
unsafe fn load_pair_u64(self, p0: *const u8, p1: *const u8) -> Self::V {
unsafe { vreinterpretq_u8_u64(vcombine_u64(vld1_u64(p0.cast()), vld1_u64(p1.cast()))) }
}
#[inline(always)]
unsafe fn load_pair_u32(self, p0: *const u8, p1: *const u8) -> Self::V {
let lo: uint8x16_t;
unsafe {
asm!(
"ldr {lo:s}, [{p0}]",
"ldr {hi:s}, [{p1}]",
"zip1 {lo:v}.2s, {lo:v}.2s, {hi:v}.2s",
p0 = in(reg) p0,
p1 = in(reg) p1,
lo = out(vreg) lo,
hi = out(vreg) _,
options(pure, readonly, nostack, preserves_flags),
);
}
lo
}
#[inline(always)]
unsafe fn u32_v(self, x: u32) -> Self::V {
unsafe { fmov_u64(x as u64) }
}
#[inline(always)]
unsafe fn xor(self, a: Self::V, b: Self::V) -> Self::V {
unsafe { veorq_u8(a, b) }
}
#[inline(always)]
unsafe fn add_u64(self, a: Self::V, b: Self::V) -> Self::V {
unsafe { vreinterpretq_u8_u64(vaddq_u64(vreinterpretq_u64_u8(a), vreinterpretq_u64_u8(b))) }
}
#[inline(always)]
unsafe fn zip_u8(self, v: Self::V) -> Self::V {
unsafe { vzip1q_u8(v, vextq_u8::<8>(v, v)) }
}
}
impl PreXorAes for Neon128 {
#[inline(always)]
unsafe fn pre_r1_zero(self, mut a: Self::V, b: Self::V) -> Self::V {
unsafe {
asm!(
"aese {a:v}.16b, {b:v}.16b\naesmc {a:v}.16b, {a:v}.16b",
a = inout(vreg) a,
b = in(vreg) b,
options(pure, nomem, nostack, preserves_flags),
);
}
a
}
#[inline(always)]
unsafe fn pre_fl_zero(self, mut a: Self::V, b: Self::V) -> Self::V {
unsafe {
asm!(
"aese {a:v}.16b, {b:v}.16b",
a = inout(vreg) a,
b = in(vreg) b,
options(pure, nomem, nostack, preserves_flags),
);
}
a
}
}
impl Lane128x8 for Neon128 {}
impl CtrKernel for Neon128 {
#[inline(always)]
unsafe fn ctr<const XOR: bool>(self, root: &[u8; 32], blk: u64, buf: &mut [u8]) {
unsafe { ctr_8x128::<Neon128, XOR>(Neon128, root, blk, buf) };
}
}
def_hw_tier!(
Neon128,
Neon128,
uint8x16_t,
all(target_feature = "aes", target_feature = "neon"),
all(target_feature = "aes", target_feature = "neon"),
any(),
"aes,neon"
);
}
#[cfg(all(target_feature = "aes", target_feature = "neon"))]
use crate::arch::tier::StaticPlatform;
#[cfg(all(target_feature = "aes", target_feature = "neon"))]
pub(crate) type ArmPlatform = StaticPlatform<hw::Tier>;
#[cfg(all(
feature = "std",
not(all(target_feature = "aes", target_feature = "neon"))
))]
mod active {
use core::sync::atomic::{AtomicUsize, Ordering};
use std::arch::is_aarch64_feature_detected;
use super::hw;
use crate::arch::{
dispatch::{Dispatch, DispatchSource, RuntimePlatform, def_runtime_dispatch, dispatch_slot},
soft::Soft128,
};
fn detect_idx() -> usize {
if is_aarch64_feature_detected!("aes") && is_aarch64_feature_detected!("neon") {
2
} else {
1
}
}
const fn build_table(init: Dispatch) -> [Dispatch; 3] {
[
init,
Dispatch::for_platform::<Soft128>(),
dispatch_slot!(hw),
]
}
def_runtime_dispatch!(ArmDispatch, 3, build_table, detect_idx);
pub(crate) type ArmPlatform = RuntimePlatform<ArmDispatch>;
}
#[cfg(all(
feature = "std",
not(all(target_feature = "aes", target_feature = "neon"))
))]
pub(crate) use active::ArmPlatform;
#[cfg(all(
not(feature = "std"),
not(all(target_feature = "aes", target_feature = "neon"))
))]
pub(crate) use crate::arch::soft::Soft128 as ArmPlatform;