mod scalar;
#[cfg(all(target_arch = "aarch64", target_endian = "little"))]
mod neon;
#[cfg(target_arch = "x86_64")]
mod x86;
#[cfg(all(target_arch = "aarch64", target_endian = "little"))]
pub(crate) use neon::Neon;
pub(crate) use scalar::Scalar;
#[cfg(target_arch = "x86_64")]
pub(crate) use x86::Avx2;
#[cfg(target_arch = "x86_64")]
pub(crate) use x86::Sse2;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum Backend {
Scalar,
Neon,
Sse2,
Avx2,
}
impl Backend {
#[must_use]
pub fn is_available(self) -> bool {
match self {
Self::Scalar => true,
Self::Neon => neon_available(),
Self::Sse2 => sse2_available(),
Self::Avx2 => avx2_available(),
}
}
}
#[must_use]
pub fn selected_backend() -> Backend {
#[cfg(all(
target_arch = "aarch64",
target_endian = "little",
target_feature = "neon"
))]
{
Backend::Neon
}
#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))]
{
Backend::Avx2
}
#[cfg(not(any(
all(
target_arch = "aarch64",
target_endian = "little",
target_feature = "neon"
),
all(target_arch = "x86_64", target_feature = "avx2")
)))]
selected_backend_fallback()
}
#[cfg(not(any(
all(
target_arch = "aarch64",
target_endian = "little",
target_feature = "neon"
),
all(target_arch = "x86_64", target_feature = "avx2")
)))]
fn selected_backend_fallback() -> Backend {
#[cfg(feature = "std")]
{
use std::sync::OnceLock;
static SELECTED: OnceLock<Backend> = OnceLock::new();
*SELECTED.get_or_init(detect_backend)
}
#[cfg(not(feature = "std"))]
detect_backend()
}
#[cfg(not(any(
all(
target_arch = "aarch64",
target_endian = "little",
target_feature = "neon"
),
all(target_arch = "x86_64", target_feature = "avx2")
)))]
fn detect_backend() -> Backend {
if avx2_available() {
Backend::Avx2
} else if sse2_available() {
Backend::Sse2
} else if neon_available() {
Backend::Neon
} else {
Backend::Scalar
}
}
#[inline]
fn neon_available() -> bool {
#[cfg(all(feature = "std", target_arch = "aarch64", target_endian = "little"))]
{
return std::arch::is_aarch64_feature_detected!("neon");
}
#[cfg(all(
not(feature = "std"),
target_arch = "aarch64",
target_endian = "little"
))]
{
return cfg!(target_feature = "neon");
}
#[allow(unreachable_code)]
false
}
#[inline]
fn sse2_available() -> bool {
#[cfg(all(feature = "std", target_arch = "x86_64"))]
{
return std::arch::is_x86_feature_detected!("sse2");
}
#[cfg(all(not(feature = "std"), target_arch = "x86_64"))]
{
return cfg!(target_feature = "sse2");
}
#[allow(unreachable_code)]
false
}
#[inline]
fn avx2_available() -> bool {
#[cfg(all(feature = "std", target_arch = "x86_64"))]
{
return std::arch::is_x86_feature_detected!("avx2");
}
#[cfg(all(not(feature = "std"), target_arch = "x86_64"))]
{
return cfg!(target_feature = "avx2");
}
#[allow(unreachable_code)]
false
}
pub(crate) trait Xxh3Kernel: Copy {
fn accumulate(self, acc: &mut [u64; 8], stripe: &[u8; 64], secret: &[u8; 64]);
fn scramble(self, acc: &mut [u64; 8], secret: &[u8; 64]);
}
macro_rules! dispatch {
($function:ident($($argument:expr),* $(,)?)) => {{
match $crate::xxhash::kernel::selected_backend() {
$crate::xxhash::kernel::Backend::Scalar => {
$function($crate::xxhash::kernel::Scalar, $($argument),*)
}
$crate::xxhash::kernel::Backend::Neon => {
#[cfg(all(target_arch = "aarch64", target_endian = "little"))]
{
$function(unsafe { $crate::xxhash::kernel::Neon::new_unchecked() }, $($argument),*)
}
#[cfg(not(all(target_arch = "aarch64", target_endian = "little")))]
unreachable!("NEON cannot be selected on this target")
}
$crate::xxhash::kernel::Backend::Sse2 => {
#[cfg(target_arch = "x86_64")]
{
$function(unsafe { $crate::xxhash::kernel::Sse2::new_unchecked() }, $($argument),*)
}
#[cfg(not(target_arch = "x86_64"))]
unreachable!("SSE2 cannot be selected on this target")
}
$crate::xxhash::kernel::Backend::Avx2 => {
#[cfg(target_arch = "x86_64")]
{
$function(unsafe { $crate::xxhash::kernel::Avx2::new_unchecked() }, $($argument),*)
}
#[cfg(not(target_arch = "x86_64"))]
unreachable!("AVX2 cannot be selected on this target")
}
}
}};
}
pub(crate) use dispatch;