use core::sync::atomic::{AtomicUsize, Ordering};
use std::arch::is_x86_feature_detected;
#[cfg(not(all(
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
)))]
use super::aes128 as aes128_mod;
#[cfg(all(
target_feature = "aes",
target_feature = "sse2",
not(all(
target_arch = "x86_64",
target_feature = "vaes",
target_feature = "avx2"
))
))]
use super::aes128::X86_128;
use super::vaes256 as vaes256_mod;
#[cfg(all(
target_arch = "x86_64",
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
))]
use super::vaes256::Vaes256;
#[cfg(target_arch = "x86_64")]
use super::vaes512;
use crate::arch::dispatch::{Dispatch, DispatchSource, def_runtime_dispatch, dispatch_slot};
#[cfg(all(target_feature = "aes", target_feature = "sse2"))]
use crate::arch::dispatch::{HybridPlatform, StaticGroup};
#[cfg(all(
target_arch = "x86_64",
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
))]
use crate::arch::wide::Wide256;
#[cfg(not(all(target_feature = "aes", target_feature = "sse2")))]
use crate::arch::{dispatch::RuntimePlatform, soft::Soft128};
#[cfg(target_arch = "x86_64")]
const HAS_AVX512_TIER: usize = 1;
#[cfg(not(target_arch = "x86_64"))]
const HAS_AVX512_TIER: usize = 0;
#[cfg(all(
target_arch = "x86_64",
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
))]
const BASE_LEVEL: usize = 3;
#[cfg(all(
not(all(
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
)),
all(
target_feature = "aes",
target_feature = "avx",
target_feature = "sse2"
)
))]
const BASE_LEVEL: usize = 2;
#[cfg(all(
not(all(
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
)),
not(all(
target_feature = "aes",
target_feature = "avx",
target_feature = "sse2"
)),
all(target_feature = "aes", target_feature = "sse2")
))]
const BASE_LEVEL: usize = 1;
#[cfg(not(all(target_feature = "aes", target_feature = "sse2")))]
const BASE_LEVEL: usize = 0;
const TABLE_LEN: usize = 5 - BASE_LEVEL + HAS_AVX512_TIER;
fn detect_raw_level() -> usize {
if is_x86_feature_detected!("aes") && is_x86_feature_detected!("sse2") {
#[cfg(target_arch = "x86_64")]
if is_x86_feature_detected!("avx512f")
&& is_x86_feature_detected!("vaes")
&& is_x86_feature_detected!("avx2")
{
return 4;
}
if is_x86_feature_detected!("vaes") && is_x86_feature_detected!("avx2") {
3
} else if is_x86_feature_detected!("avx") {
2
} else {
1
}
} else {
0
}
}
fn detect_idx() -> usize {
1 + detect_raw_level().saturating_sub(BASE_LEVEL)
}
const fn build_table(init: Dispatch) -> [Dispatch; TABLE_LEN] {
[
init,
#[cfg(not(all(target_feature = "aes", target_feature = "sse2")))]
Dispatch::for_platform::<Soft128>(),
#[cfg(all(
not(all(
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
)),
not(all(
target_feature = "aes",
target_feature = "avx",
target_feature = "sse2"
))
))]
dispatch_slot!(aes128_mod::sse2_aes),
#[cfg(not(all(
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
)))]
dispatch_slot!(aes128_mod::avx_aes),
dispatch_slot!(vaes256_mod),
#[cfg(target_arch = "x86_64")]
dispatch_slot!(vaes512),
]
}
def_runtime_dispatch!(X86Dispatch, TABLE_LEN, build_table, detect_idx);
#[cfg(all(target_feature = "aes", target_feature = "sse2"))]
pub(crate) struct InlineShort;
#[cfg(all(
target_feature = "aes",
target_feature = "sse2",
not(all(
target_arch = "x86_64",
target_feature = "vaes",
target_feature = "avx2"
))
))]
unsafe impl StaticGroup for InlineShort {
type G = X86_128;
#[inline(always)]
fn group() -> Self::G {
X86_128
}
}
#[cfg(all(
target_arch = "x86_64",
target_feature = "vaes",
target_feature = "avx2",
target_feature = "aes",
target_feature = "sse2"
))]
unsafe impl StaticGroup for InlineShort {
type G = Wide256<Vaes256>;
#[inline(always)]
fn group() -> Self::G {
Wide256(Vaes256)
}
}
#[cfg(not(all(target_feature = "aes", target_feature = "sse2")))]
pub(crate) type X86Platform = RuntimePlatform<X86Dispatch>;
#[cfg(all(target_feature = "aes", target_feature = "sse2"))]
pub(crate) type X86Platform = HybridPlatform<X86Dispatch, InlineShort>;