use core::sync::atomic::{AtomicU8, Ordering};
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
#[repr(u8)]
pub enum Level {
Scalar = 0,
Sse2 = 1,
Ssse3 = 2,
Sse41 = 3,
Neon = 4,
Wasm = 5,
Avx2 = 6,
Avx512 = 7,
}
static DETECTED: AtomicU8 = AtomicU8::new(255);
pub fn detected_level() -> Level {
let cached = DETECTED.load(Ordering::Relaxed);
if cached != 255 {
return from_byte(cached);
}
let level = detect();
DETECTED.store(level as u8, Ordering::Relaxed);
level
}
fn from_byte(n: u8) -> Level {
match n {
1 => Level::Sse2,
2 => Level::Ssse3,
3 => Level::Sse41,
4 => Level::Neon,
5 => Level::Wasm,
6 => Level::Avx2,
7 => Level::Avx512,
_ => Level::Scalar,
}
}
#[cfg(any(test, feature = "parity-internals"))]
fn is_x86(level: Level) -> bool {
matches!(
level,
Level::Sse2 | Level::Ssse3 | Level::Sse41 | Level::Avx2 | Level::Avx512
)
}
fn detect() -> Level {
#[cfg(target_arch = "x86_64")]
{
#[cfg(feature = "std")]
let (shuffle, meshlet) = (
std::is_x86_feature_detected!("ssse3") && std::is_x86_feature_detected!("popcnt"),
std::is_x86_feature_detected!("sse4.1"),
);
#[cfg(not(feature = "std"))]
let (shuffle, meshlet) = (
cfg!(all(target_feature = "ssse3", target_feature = "popcnt")),
cfg!(target_feature = "sse4.1"),
);
#[cfg(feature = "std")]
let (avx2, avx512) = (
std::is_x86_feature_detected!("avx2"),
std::is_x86_feature_detected!("avx512f")
&& std::is_x86_feature_detected!("avx512bw")
&& std::is_x86_feature_detected!("avx512vl")
&& std::is_x86_feature_detected!("avx512vbmi")
&& std::is_x86_feature_detected!("avx512vbmi2"),
);
#[cfg(not(feature = "std"))]
let (avx2, avx512) = (
cfg!(target_feature = "avx2"),
cfg!(all(
target_feature = "avx512f",
target_feature = "avx512bw",
target_feature = "avx512vl",
target_feature = "avx512vbmi",
target_feature = "avx512vbmi2"
)),
);
return if shuffle && meshlet && avx2 && avx512 {
Level::Avx512
} else if shuffle && meshlet && avx2 {
Level::Avx2
} else if shuffle && meshlet {
Level::Sse41
} else if shuffle {
Level::Ssse3
} else {
Level::Sse2
};
}
#[cfg(target_arch = "aarch64")]
{
return Level::Neon;
}
#[cfg(all(target_arch = "wasm32", target_feature = "simd128"))]
{
return Level::Wasm;
}
#[allow(unreachable_code)]
Level::Scalar
}
#[cfg(all(feature = "std", any(test, feature = "parity-internals")))]
std::thread_local! { static LIMIT: core::cell::Cell<u8> = const { core::cell::Cell::new(255) }; }
#[cfg(all(not(feature = "std"), any(test, feature = "parity-internals")))]
static LIMIT: AtomicU8 = AtomicU8::new(255);
#[cfg(any(
target_arch = "x86_64",
target_arch = "aarch64",
all(target_arch = "wasm32", target_feature = "simd128")
))]
pub(super) fn selected() -> Level {
let detected = detected_level();
#[cfg(any(test, feature = "parity-internals"))]
{
#[cfg(feature = "std")]
let limit = LIMIT.with(|l| l.get());
#[cfg(not(feature = "std"))]
let limit = LIMIT.load(Ordering::Relaxed);
if limit == 0 {
return Level::Scalar;
}
if is_x86(detected) && is_x86(from_byte(limit)) {
return from_byte((detected as u8).min(limit));
}
}
detected
}
#[cfg(any(test, feature = "parity-internals"))]
pub fn with_level<R>(level: Level, f: impl FnOnce() -> R) -> Result<R, crate::Error> {
let detected = detected_level();
if level != Level::Scalar
&& !(level == detected || (is_x86(detected) && is_x86(level) && level <= detected))
{
return Err(crate::Error::InvalidParameter);
}
struct Restore(u8);
impl Drop for Restore {
fn drop(&mut self) {
#[cfg(feature = "std")]
LIMIT.with(|l| l.set(self.0));
#[cfg(not(feature = "std"))]
LIMIT.store(self.0, Ordering::Relaxed);
}
}
#[cfg(feature = "std")]
let old = LIMIT.with(|l| l.replace(level as u8));
#[cfg(not(feature = "std"))]
let old = LIMIT.swap(level as u8, Ordering::Relaxed);
let _restore = Restore(old);
Ok(f())
}
#[cfg(any(
target_arch = "x86_64",
target_arch = "aarch64",
all(target_arch = "wasm32", target_feature = "simd128")
))]
pub(super) struct Baseline(());
#[cfg(target_arch = "x86_64")]
pub(super) struct Ssse3(());
#[cfg(target_arch = "x86_64")]
pub(super) struct Sse41(());
#[cfg(any(
target_arch = "x86_64",
target_arch = "aarch64",
all(target_arch = "wasm32", target_feature = "simd128")
))]
pub(super) fn baseline() -> Option<Baseline> {
(selected() != Level::Scalar).then_some(Baseline(()))
}
#[cfg(target_arch = "x86_64")]
pub(super) fn ssse3() -> Option<Ssse3> {
matches!(
selected(),
Level::Ssse3 | Level::Sse41 | Level::Avx2 | Level::Avx512
)
.then_some(Ssse3(()))
}
#[cfg(target_arch = "x86_64")]
pub(super) fn sse41() -> Option<Sse41> {
matches!(selected(), Level::Sse41 | Level::Avx2 | Level::Avx512).then_some(Sse41(()))
}
#[cfg(target_arch = "x86_64")]
pub(super) struct Avx512(());
#[cfg(target_arch = "x86_64")]
pub(super) fn avx512() -> Option<Avx512> {
(selected() == Level::Avx512).then_some(Avx512(()))
}