use std::fmt;
use std::sync::LazyLock;
mod dispatch;
pub use dispatch::{Candidate, Dispatch, Kernel, Selection};
#[derive(Clone, Copy, PartialEq, Eq, Hash, Default)]
pub struct CpuFeatures(u32);
impl CpuFeatures {
pub const SSE2: Self = Self(1 << 0);
pub const SSE3: Self = Self(1 << 1);
pub const SSSE3: Self = Self(1 << 2);
pub const SSE4_1: Self = Self(1 << 3);
pub const SSE4_2: Self = Self(1 << 4);
pub const POPCNT: Self = Self(1 << 5);
pub const AVX: Self = Self(1 << 6);
pub const AVX2: Self = Self(1 << 7);
pub const FMA: Self = Self(1 << 8);
pub const F16C: Self = Self(1 << 9);
pub const BMI1: Self = Self(1 << 10);
pub const BMI2: Self = Self(1 << 11);
pub const AVX512F: Self = Self(1 << 12);
pub const AVX512BW: Self = Self(1 << 13);
pub const AVX512CD: Self = Self(1 << 14);
pub const AVX512DQ: Self = Self(1 << 15);
pub const AVX512VL: Self = Self(1 << 16);
pub const AVX512FP16: Self = Self(1 << 17);
pub const NEON: Self = Self(1 << 18);
pub const FP16: Self = Self(1 << 19);
pub const LSX: Self = Self(1 << 20);
pub const LASX: Self = Self(1 << 21);
pub const fn empty() -> Self {
Self(0)
}
pub const fn is_empty(self) -> bool {
self.0 == 0
}
pub const fn union(self, other: Self) -> Self {
Self(self.0 | other.0)
}
pub const fn contains(self, other: Self) -> bool {
self.0 & other.0 == other.0
}
pub fn host() -> Self {
*HOST_FEATURES
}
pub const fn baseline() -> Self {
BASELINE
}
pub const fn from_target_feature(name: &str) -> Option<Self> {
let mut i = 0;
while i < FEATURE_TABLE.len() {
if str_eq(FEATURE_TABLE[i].name, name) {
return Some(FEATURE_TABLE[i].feature.with_implied());
}
i += 1;
}
None
}
pub const fn from_target_features(names: &[&str]) -> Self {
let mut features = Self::empty();
let mut i = 0;
while i < names.len() {
match Self::from_target_feature(names[i]) {
Some(feature) => features = features.union(feature),
None => panic!("unknown target feature; add it to CpuFeatures"),
}
i += 1;
}
features
}
#[cfg(test)]
pub fn all() -> impl Iterator<Item = Self> {
FEATURE_TABLE.iter().map(|info| info.feature)
}
pub fn names(self) -> impl Iterator<Item = &'static str> {
FEATURE_TABLE
.iter()
.filter(move |info| self.contains(info.feature))
.map(|info| info.name)
}
fn x86_names(self) -> impl Iterator<Item = &'static str> {
FEATURE_TABLE
.iter()
.filter(move |info| info.is_x86 && self.contains(info.feature))
.map(|info| info.name)
}
const fn with_implied(self) -> Self {
let mut features = self;
loop {
let mut expanded = features;
let mut i = 0;
while i < FEATURE_TABLE.len() {
if features.contains(FEATURE_TABLE[i].feature) {
expanded = expanded.union(FEATURE_TABLE[i].implies);
}
i += 1;
}
if expanded.0 == features.0 {
return features;
}
features = expanded;
}
}
}
impl fmt::Debug for CpuFeatures {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_set().entries(self.names()).finish()
}
}
struct FeatureInfo {
feature: CpuFeatures,
name: &'static str,
is_x86: bool,
implies: CpuFeatures,
}
const fn x86(feature: CpuFeatures, name: &'static str, implies: CpuFeatures) -> FeatureInfo {
FeatureInfo {
feature,
name,
is_x86: true,
implies,
}
}
const fn other(feature: CpuFeatures, name: &'static str, implies: CpuFeatures) -> FeatureInfo {
FeatureInfo {
feature,
name,
is_x86: false,
implies,
}
}
const FEATURE_TABLE: &[FeatureInfo] = {
use CpuFeatures as F;
&[
x86(F::SSE2, "sse2", F::empty()),
x86(F::SSE3, "sse3", F::SSE2),
x86(F::SSSE3, "ssse3", F::SSE3),
x86(F::SSE4_1, "sse4.1", F::SSSE3),
x86(F::SSE4_2, "sse4.2", F::SSE4_1),
x86(F::POPCNT, "popcnt", F::empty()),
x86(F::AVX, "avx", F::SSE4_2),
x86(F::AVX2, "avx2", F::AVX),
x86(F::FMA, "fma", F::AVX),
x86(F::F16C, "f16c", F::AVX),
x86(F::BMI1, "bmi1", F::empty()),
x86(F::BMI2, "bmi2", F::empty()),
x86(F::AVX512F, "avx512f", F::AVX2.union(F::FMA).union(F::F16C)),
x86(F::AVX512BW, "avx512bw", F::AVX512F),
x86(F::AVX512CD, "avx512cd", F::AVX512F),
x86(F::AVX512DQ, "avx512dq", F::AVX512F),
x86(F::AVX512VL, "avx512vl", F::AVX512F),
x86(
F::AVX512FP16,
"avx512fp16",
F::AVX512BW.union(F::AVX512DQ).union(F::AVX512VL),
),
other(F::NEON, "neon", F::empty()),
other(F::FP16, "fp16", F::NEON),
other(F::LSX, "lsx", F::empty()),
other(F::LASX, "lasx", F::LSX),
]
};
const fn str_eq(a: &str, b: &str) -> bool {
let (a, b) = (a.as_bytes(), b.as_bytes());
if a.len() != b.len() {
return false;
}
let mut i = 0;
while i < a.len() {
if a[i] != b[i] {
return false;
}
i += 1;
}
true
}
const BASELINE: CpuFeatures = {
let enabled = [
(cfg!(target_feature = "sse2"), CpuFeatures::SSE2),
(cfg!(target_feature = "sse3"), CpuFeatures::SSE3),
(cfg!(target_feature = "ssse3"), CpuFeatures::SSSE3),
(cfg!(target_feature = "sse4.1"), CpuFeatures::SSE4_1),
(cfg!(target_feature = "sse4.2"), CpuFeatures::SSE4_2),
(cfg!(target_feature = "popcnt"), CpuFeatures::POPCNT),
(cfg!(target_feature = "avx"), CpuFeatures::AVX),
(cfg!(target_feature = "avx2"), CpuFeatures::AVX2),
(cfg!(target_feature = "fma"), CpuFeatures::FMA),
(cfg!(target_feature = "f16c"), CpuFeatures::F16C),
(cfg!(target_feature = "bmi1"), CpuFeatures::BMI1),
(cfg!(target_feature = "bmi2"), CpuFeatures::BMI2),
(cfg!(target_feature = "avx512f"), CpuFeatures::AVX512F),
(cfg!(target_feature = "avx512bw"), CpuFeatures::AVX512BW),
(cfg!(target_feature = "avx512cd"), CpuFeatures::AVX512CD),
(cfg!(target_feature = "avx512dq"), CpuFeatures::AVX512DQ),
(cfg!(target_feature = "avx512vl"), CpuFeatures::AVX512VL),
(cfg!(target_feature = "avx512fp16"), CpuFeatures::AVX512FP16),
(cfg!(target_feature = "neon"), CpuFeatures::NEON),
(cfg!(target_feature = "fp16"), CpuFeatures::FP16),
(cfg!(target_feature = "lsx"), CpuFeatures::LSX),
(cfg!(target_feature = "lasx"), CpuFeatures::LASX),
];
let mut features = CpuFeatures::empty();
let mut i = 0;
while i < enabled.len() {
if enabled[i].0 {
features = features.union(enabled[i].1);
}
i += 1;
}
features
};
static HOST_FEATURES: LazyLock<CpuFeatures> =
LazyLock::new(|| detect_host_features().union(BASELINE));
#[cfg(target_arch = "x86_64")]
fn detect_host_features() -> CpuFeatures {
let mut features = CpuFeatures::empty();
let mut add = |is_detected: bool, feature: CpuFeatures| {
if is_detected {
features = features.union(feature);
}
};
add(is_x86_feature_detected!("sse2"), CpuFeatures::SSE2);
add(is_x86_feature_detected!("sse3"), CpuFeatures::SSE3);
add(is_x86_feature_detected!("ssse3"), CpuFeatures::SSSE3);
add(is_x86_feature_detected!("sse4.1"), CpuFeatures::SSE4_1);
add(is_x86_feature_detected!("sse4.2"), CpuFeatures::SSE4_2);
add(is_x86_feature_detected!("popcnt"), CpuFeatures::POPCNT);
add(is_x86_feature_detected!("avx"), CpuFeatures::AVX);
add(is_x86_feature_detected!("avx2"), CpuFeatures::AVX2);
add(is_x86_feature_detected!("fma"), CpuFeatures::FMA);
add(is_x86_feature_detected!("f16c"), CpuFeatures::F16C);
add(is_x86_feature_detected!("bmi1"), CpuFeatures::BMI1);
add(is_x86_feature_detected!("bmi2"), CpuFeatures::BMI2);
add(is_x86_feature_detected!("avx512f"), CpuFeatures::AVX512F);
add(is_x86_feature_detected!("avx512bw"), CpuFeatures::AVX512BW);
add(is_x86_feature_detected!("avx512cd"), CpuFeatures::AVX512CD);
add(is_x86_feature_detected!("avx512dq"), CpuFeatures::AVX512DQ);
add(is_x86_feature_detected!("avx512vl"), CpuFeatures::AVX512VL);
add(
is_x86_feature_detected!("avx512fp16"),
CpuFeatures::AVX512FP16,
);
features
}
#[cfg(target_arch = "aarch64")]
fn detect_host_features() -> CpuFeatures {
#[cfg(any(target_os = "ios", target_os = "tvos"))]
let has_fp16 = true;
#[cfg(not(any(target_os = "ios", target_os = "tvos")))]
let has_fp16 = aarch64::has_neon_f16_support();
if has_fp16 {
CpuFeatures::NEON.union(CpuFeatures::FP16)
} else {
CpuFeatures::NEON
}
}
#[cfg(target_arch = "loongarch64")]
fn detect_host_features() -> CpuFeatures {
let mut features = CpuFeatures::empty();
if loongarch64::has_lsx_support() {
features = features.union(CpuFeatures::LSX);
}
if loongarch64::has_lasx_support() {
features = features.union(CpuFeatures::LASX);
}
features
}
#[cfg(not(any(
target_arch = "aarch64",
target_arch = "x86_64",
target_arch = "loongarch64"
)))]
fn detect_host_features() -> CpuFeatures {
CpuFeatures::empty()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum SimdSupport {
None,
Neon,
Sse,
Avx,
AvxFma,
Avx2,
Avx512,
Avx512FP16,
Lsx,
Lasx,
}
impl fmt::Display for SimdSupport {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let name = match self {
Self::None => "none",
Self::Neon => "neon",
Self::Sse => "sse",
Self::Avx => "avx",
Self::AvxFma => "avx_fma",
Self::Avx2 => "avx2",
Self::Avx512 => "avx512",
Self::Avx512FP16 => "avx512_fp16",
Self::Lsx => "lsx",
Self::Lasx => "lasx",
};
f.write_str(name)
}
}
#[derive(Debug, Clone)]
pub struct SimdInfo {
pub tier: SimdSupport,
pub target_arch: &'static str,
pub host_features: Vec<&'static str>,
}
pub fn simd_info() -> SimdInfo {
SimdInfo {
tier: *SIMD_SUPPORT,
target_arch: std::env::consts::ARCH,
host_features: CpuFeatures::host().x86_names().collect(),
}
}
impl SimdSupport {
fn from_features(features: CpuFeatures) -> Self {
use CpuFeatures as F;
if features.contains(F::AVX512F) {
if features.contains(F::AVX512FP16) {
Self::Avx512FP16
} else {
Self::Avx512
}
} else if features.contains(F::AVX2.union(F::FMA)) {
Self::Avx2
} else if features.contains(F::AVX.union(F::FMA)) {
Self::AvxFma
} else if features.contains(F::AVX) {
Self::Avx
} else if features.contains(F::FP16) {
Self::Neon
} else if features.contains(F::LASX) {
Self::Lasx
} else if features.contains(F::LSX) {
Self::Lsx
} else {
Self::None
}
}
}
pub static SIMD_SUPPORT: LazyLock<SimdSupport> =
LazyLock::new(|| SimdSupport::from_features(CpuFeatures::host()));
#[cfg(all(target_arch = "aarch64", target_os = "macos"))]
mod aarch64 {
pub fn has_neon_f16_support() -> bool {
true
}
}
#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
mod aarch64 {
pub fn has_neon_f16_support() -> bool {
let flags = unsafe { libc::getauxval(libc::AT_HWCAP) };
flags & libc::HWCAP_FPHP != 0
}
}
#[cfg(all(target_arch = "aarch64", target_os = "windows"))]
mod aarch64 {
pub fn has_neon_f16_support() -> bool {
false
}
}
#[cfg(target_arch = "loongarch64")]
mod loongarch64 {
pub fn has_lsx_support() -> bool {
let flags = unsafe { libc::getauxval(libc::AT_HWCAP) };
flags & libc::HWCAP_LOONGARCH_LSX != 0
}
pub fn has_lasx_support() -> bool {
let flags = unsafe { libc::getauxval(libc::AT_HWCAP) };
flags & libc::HWCAP_LOONGARCH_LASX != 0
}
}
#[cfg(all(target_arch = "aarch64", target_os = "android"))]
mod aarch64 {
pub fn has_neon_f16_support() -> bool {
false
}
}
#[cfg(test)]
mod tests {
use super::*;
use rstest::rstest;
#[test]
fn simd_info_exposes_tier() {
let info = simd_info();
assert_eq!(info.target_arch, std::env::consts::ARCH);
assert_eq!(info.tier, *SIMD_SUPPORT);
}
#[cfg(target_arch = "x86_64")]
#[test]
fn simd_info_features_include_baseline() {
let info = simd_info();
assert!(info.host_features.contains(&"sse2"));
}
#[cfg(not(target_arch = "x86_64"))]
#[test]
fn simd_info_features_empty_off_x86_64() {
let info = simd_info();
assert!(info.host_features.is_empty());
}
#[cfg(target_arch = "x86_64")]
#[test]
fn avx_fma_tiers_are_only_selected_when_fma_is_detected() {
if matches!(*SIMD_SUPPORT, SimdSupport::Avx2 | SimdSupport::AvxFma) {
assert!(
is_x86_feature_detected!("fma"),
"tier {} dispatches to avx,fma kernels but the host has no FMA",
*SIMD_SUPPORT
);
}
}
#[test]
fn host_features_include_build_baseline() {
assert!(CpuFeatures::host().contains(CpuFeatures::baseline()));
}
#[rstest]
#[case::sse_chain("avx", CpuFeatures::SSE2.union(CpuFeatures::SSE4_2))]
#[case::fma_implies_avx("fma", CpuFeatures::AVX)]
#[case::avx512f(
"avx512f",
CpuFeatures::AVX2.union(CpuFeatures::FMA).union(CpuFeatures::F16C)
)]
#[case::avx512fp16("avx512fp16", CpuFeatures::AVX512BW.union(CpuFeatures::AVX512F))]
#[case::fp16("fp16", CpuFeatures::NEON)]
#[case::lasx("lasx", CpuFeatures::LSX)]
fn target_feature_mask_includes_implied_features(
#[case] name: &str,
#[case] implied: CpuFeatures,
) {
let features = CpuFeatures::from_target_feature(name).unwrap();
assert!(
features.contains(implied),
"{name} -> {features:?} is missing {implied:?}"
);
}
#[test]
fn unknown_target_feature_is_rejected() {
assert_eq!(CpuFeatures::from_target_feature("avx9000"), None);
}
#[rstest]
#[case::none(&[], SimdSupport::None)]
#[case::avx(&["avx"], SimdSupport::Avx)]
#[case::avx2_without_fma(&["avx2"], SimdSupport::Avx)]
#[case::avx_fma(&["fma"], SimdSupport::AvxFma)]
#[case::avx2(&["avx2", "fma"], SimdSupport::Avx2)]
#[case::avx512(&["avx512f"], SimdSupport::Avx512)]
#[case::avx512_fp16(&["avx512fp16"], SimdSupport::Avx512FP16)]
#[case::neon_without_fp16(&["neon"], SimdSupport::None)]
#[case::neon(&["fp16"], SimdSupport::Neon)]
#[case::lsx(&["lsx"], SimdSupport::Lsx)]
#[case::lasx(&["lasx"], SimdSupport::Lasx)]
fn simd_support_is_derived_from_features(
#[case] names: &[&str],
#[case] expected: SimdSupport,
) {
let features = CpuFeatures::from_target_features(names);
assert_eq!(SimdSupport::from_features(features), expected);
}
#[rstest]
#[case::none(SimdSupport::None, "none")]
#[case::neon(SimdSupport::Neon, "neon")]
#[case::sse(SimdSupport::Sse, "sse")]
#[case::avx(SimdSupport::Avx, "avx")]
#[case::avx_fma(SimdSupport::AvxFma, "avx_fma")]
#[case::avx2(SimdSupport::Avx2, "avx2")]
#[case::avx512(SimdSupport::Avx512, "avx512")]
#[case::avx512_fp16(SimdSupport::Avx512FP16, "avx512_fp16")]
#[case::lsx(SimdSupport::Lsx, "lsx")]
#[case::lasx(SimdSupport::Lasx, "lasx")]
fn simd_support_display_matches_lowercase_convention(
#[case] tier: SimdSupport,
#[case] expected: &str,
) {
assert_eq!(tier.to_string(), expected);
}
}