use std::sync::OnceLock;
use super::bitmask::VECTOR_BYTES;
use super::lanes::CompileTimeLanes;
pub(super) trait Isa: Copy {
type Lanes: Copy;
fn load(self, input: &[u8; VECTOR_BYTES]) -> Self::Lanes;
fn eq_mask(self, v: Self::Lanes, b: u8) -> u64;
fn xor_prefix_sum(self, q: u64) -> u64;
fn pext(self, src: u64, mask: u64) -> u64;
const FAST_PEXT: bool;
fn vectorize<R>(self, f: impl FnOnce() -> R) -> R;
}
#[derive(Debug, Default, Clone, Copy)]
pub(super) struct CompileTime;
impl Isa for CompileTime {
type Lanes = CompileTimeLanes;
const FAST_PEXT: bool = cfg!(all(target_arch = "x86_64", target_feature = "bmi2"));
#[inline(always)]
fn load(self, input: &[u8; VECTOR_BYTES]) -> CompileTimeLanes {
unsafe { CompileTimeLanes::load(input) }
}
#[inline(always)]
fn eq_mask(self, v: CompileTimeLanes, b: u8) -> u64 {
unsafe { v.eq_mask(b) }
}
#[inline(always)]
fn xor_prefix_sum(self, q: u64) -> u64 {
super::bitmask::xor_prefix_sum(q)
}
#[inline(always)]
fn pext(self, src: u64, mask: u64) -> u64 {
super::bitops::pext(src, mask)
}
#[inline(always)]
fn vectorize<R>(self, f: impl FnOnce() -> R) -> R {
f()
}
}
macro_rules! x86_token {
(
$(#[$doc:meta])*
$name:ident, $detect:ident, lanes: $lanes:ident,
features: $features:literal, detect: [$($feature:tt),+ $(,)?],
) => {
$(#[$doc])*
#[cfg(all(target_arch = "x86_64", not(all($(target_feature = $feature),+))))]
#[derive(Debug, Clone, Copy)]
pub(super) struct $name(());
$(#[$doc])*
#[cfg(all(target_arch = "x86_64", all($(target_feature = $feature),+)))]
pub(super) type $name = CompileTime;
#[cfg(all(target_arch = "x86_64", not(all($(target_feature = $feature),+))))]
fn $detect() -> Option<$name> {
(true $(&& std::arch::is_x86_feature_detected!($feature))+).then_some($name(()))
}
#[cfg(all(target_arch = "x86_64", all($(target_feature = $feature),+)))]
fn $detect() -> Option<$name> {
None
}
#[cfg(all(target_arch = "x86_64", not(all($(target_feature = $feature),+))))]
impl Isa for $name {
type Lanes = super::lanes::$lanes;
const FAST_PEXT: bool = true;
#[inline(always)]
fn load(self, input: &[u8; VECTOR_BYTES]) -> Self::Lanes {
unsafe { super::lanes::$lanes::load(input) }
}
#[inline(always)]
fn eq_mask(self, v: Self::Lanes, b: u8) -> u64 {
unsafe { v.eq_mask(b) }
}
#[inline(always)]
fn xor_prefix_sum(self, q: u64) -> u64 {
unsafe { super::bitmask::xor_prefix_sum_clmul(q) }
}
#[inline(always)]
fn pext(self, src: u64, mask: u64) -> u64 {
unsafe { super::bitops::pext_bmi2(src, mask) }
}
#[inline(always)]
fn vectorize<R>(self, f: impl FnOnce() -> R) -> R {
#[target_feature(enable = $features)]
fn run<R>(f: impl FnOnce() -> R) -> R {
f()
}
unsafe { run(f) }
}
}
#[cfg(all(test, target_arch = "x86_64", not(all($(target_feature = $feature),+))))]
mod $detect {
#[test]
fn enables_exactly_what_it_detects() {
let enabled: Vec<&str> = $features.split(',').collect();
assert_eq!(enabled, [$($feature),+]);
}
}
};
}
x86_token! {
Avx2, detect_avx2, lanes: Avx2Lanes,
features: "avx2,bmi1,bmi2,lzcnt,popcnt,pclmulqdq",
detect: ["avx2", "bmi1", "bmi2", "lzcnt", "popcnt", "pclmulqdq"],
}
x86_token! {
Avx512, detect_avx512, lanes: Avx512Lanes,
features: "avx2,bmi1,bmi2,lzcnt,popcnt,pclmulqdq,avx512f,avx512bw,avx512vl",
detect: [
"avx2", "bmi1", "bmi2", "lzcnt", "popcnt", "pclmulqdq",
"avx512f", "avx512bw", "avx512vl",
],
}
#[cfg(all(target_arch = "aarch64", not(target_feature = "aes")))]
#[derive(Debug, Clone, Copy)]
pub(super) struct Pmull(());
#[cfg(all(target_arch = "aarch64", target_feature = "aes"))]
pub(super) type Pmull = CompileTime;
#[cfg(all(target_arch = "aarch64", not(target_feature = "aes")))]
fn detect_pmull() -> Option<Pmull> {
std::arch::is_aarch64_feature_detected!("aes").then_some(Pmull(()))
}
#[cfg(all(target_arch = "aarch64", target_feature = "aes"))]
fn detect_pmull() -> Option<Pmull> {
None
}
#[cfg(all(target_arch = "aarch64", not(target_feature = "aes")))]
impl Isa for Pmull {
type Lanes = CompileTimeLanes;
const FAST_PEXT: bool = false;
#[inline(always)]
fn load(self, input: &[u8; VECTOR_BYTES]) -> CompileTimeLanes {
unsafe { CompileTimeLanes::load(input) }
}
#[inline(always)]
fn eq_mask(self, v: CompileTimeLanes, b: u8) -> u64 {
unsafe { v.eq_mask(b) }
}
#[inline(always)]
fn xor_prefix_sum(self, q: u64) -> u64 {
unsafe { super::bitmask::xor_prefix_sum_pmull(q) }
}
#[inline(always)]
fn pext(self, src: u64, mask: u64) -> u64 {
super::bitops::pext(src, mask)
}
#[inline(always)]
fn vectorize<R>(self, f: impl FnOnce() -> R) -> R {
#[target_feature(enable = "aes")]
fn run<R>(f: impl FnOnce() -> R) -> R {
f()
}
unsafe { run(f) }
}
}
#[derive(Debug, Clone, Copy)]
pub(super) enum Level {
CompileTime(CompileTime),
#[cfg(target_arch = "x86_64")]
Avx2(Avx2),
#[cfg(target_arch = "x86_64")]
Avx512(Avx512),
#[cfg(target_arch = "aarch64")]
Pmull(Pmull),
}
impl Level {
pub(super) fn get() -> Self {
static LEVEL: OnceLock<Level> = OnceLock::new();
*LEVEL.get_or_init(|| {
let pinned = std::env::var("CSVEEE_SIMD_LEVEL").ok();
let available = Self::available();
pinned
.and_then(|name| available.iter().copied().find(|l| l.name() == name))
.unwrap_or(available[0])
})
}
fn available() -> Vec<Self> {
[
#[cfg(target_arch = "x86_64")]
detect_avx512().map(Level::Avx512),
#[cfg(target_arch = "x86_64")]
detect_avx2().map(Level::Avx2),
#[cfg(target_arch = "aarch64")]
detect_pmull().map(Level::Pmull),
Some(Level::CompileTime(CompileTime)),
]
.into_iter()
.flatten()
.collect()
}
pub(super) fn name(self) -> &'static str {
match self {
Level::CompileTime(_) => "compile-time",
#[cfg(target_arch = "x86_64")]
Level::Avx2(_) => "avx2",
#[cfg(target_arch = "x86_64")]
Level::Avx512(_) => "avx512",
#[cfg(target_arch = "aarch64")]
Level::Pmull(_) => "pmull",
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn level_is_available_and_compile_time_is_last() {
let available = Level::available();
assert!(matches!(available.last(), Some(Level::CompileTime(_))));
let level = Level::get();
assert!(available.iter().any(|l| l.name() == level.name()));
eprintln!(
"SIMD level: {} (available: {:?})",
level.name(),
available.iter().map(|l| l.name()).collect::<Vec<_>>()
);
}
}