use std::sync::OnceLock;
pub(crate) trait Block {
fn run_block(&self, phase: usize, block: usize);
}
pub(crate) struct OneCopy<F: Fn(usize, usize)>(pub(crate) F);
impl<F: Fn(usize, usize)> Block for OneCopy<F> {
fn run_block(&self, phase: usize, block: usize) {
(self.0)(phase, block);
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub(crate) enum Level {
Base,
V2,
V3,
V4,
}
impl Level {
#[cfg(test)]
pub(crate) const ALL: [Level; 4] = [Level::Base, Level::V2, Level::V3, Level::V4];
fn cap_named(value: &str) -> Option<Level> {
match value.trim() {
"x86-64" => Some(Level::Base),
"x86-64-v2" => Some(Level::V2),
"x86-64-v3" => Some(Level::V3),
"x86-64-v4" => Some(Level::V4),
_no_cap => None,
}
}
}
fn cap() -> Option<Level> {
match std::env::var("PWRS_CPU_MAX") {
Ok(value) => Level::cap_named(&value),
Err(std::env::VarError::NotPresent) => None,
Err(std::env::VarError::NotUnicode(_value)) => None,
}
}
#[cfg(target_arch = "x86_64")]
fn offered() -> Level {
let v2 = is_x86_feature_detected!("sse3")
&& is_x86_feature_detected!("ssse3")
&& is_x86_feature_detected!("sse4.1")
&& is_x86_feature_detected!("sse4.2")
&& is_x86_feature_detected!("popcnt")
&& is_x86_feature_detected!("cmpxchg16b");
let v3 = v2
&& is_x86_feature_detected!("avx")
&& is_x86_feature_detected!("avx2")
&& is_x86_feature_detected!("bmi1")
&& is_x86_feature_detected!("bmi2")
&& is_x86_feature_detected!("f16c")
&& is_x86_feature_detected!("fma")
&& is_x86_feature_detected!("lzcnt")
&& is_x86_feature_detected!("movbe")
&& is_x86_feature_detected!("xsave");
let v4 = v3
&& is_x86_feature_detected!("avx512f")
&& is_x86_feature_detected!("avx512bw")
&& is_x86_feature_detected!("avx512cd")
&& is_x86_feature_detected!("avx512dq")
&& is_x86_feature_detected!("avx512vl");
if v4 {
Level::V4
} else if v3 {
Level::V3
} else if v2 {
Level::V2
} else {
Level::Base
}
}
#[cfg(not(target_arch = "x86_64"))]
fn offered() -> Level {
Level::Base
}
pub(crate) fn process_level() -> Level {
static LEVEL: OnceLock<Level> = OnceLock::new();
*LEVEL.get_or_init(|| match cap() {
Some(cap) => offered().min(cap),
None => offered(),
})
}
#[cfg(not(test))]
pub(crate) fn for_new_job() -> Level {
process_level()
}
#[cfg(test)]
thread_local! {
static FORCED: std::cell::Cell<Option<Level>> = const { std::cell::Cell::new(None) };
}
#[cfg(test)]
pub(crate) fn for_new_job() -> Level {
FORCED
.with(std::cell::Cell::get)
.unwrap_or_else(process_level)
}
#[cfg(test)]
pub(crate) fn levels_offered() -> Vec<Level> {
let top = offered();
Level::ALL.into_iter().filter(|&l| l <= top).collect()
}
#[cfg(test)]
pub(crate) fn with_level<R>(level: Level, f: impl FnOnce() -> R) -> R {
assert!(
level <= offered(),
"{level:?} is above what this CPU offers"
);
let before = FORCED.with(|forced| forced.replace(Some(level)));
let answer = f();
FORCED.with(|forced| forced.set(before));
answer
}
#[cfg(test)]
pub(crate) fn assert_every_level_agrees<T: PartialEq>(what: &str, answer: impl Fn() -> T) {
let base = with_level(Level::Base, &answer);
for level in levels_offered() {
assert!(
with_level(level, &answer) == base,
"{what} at {level:?} answered other bits than at the baseline"
);
}
}
#[cfg(target_arch = "x86_64")]
#[inline]
pub(crate) fn run_at<B: Block>(level: Level, work: &B, phase: usize, block: usize) {
match level {
Level::V4 => unsafe { v4(work, phase, block) },
Level::V3 => unsafe { v3(work, phase, block) },
Level::V2 => unsafe { v2(work, phase, block) },
Level::Base => work.run_block(phase, block),
}
}
#[cfg(not(target_arch = "x86_64"))]
#[inline]
pub(crate) fn run_at<B: Block>(_level: Level, work: &B, phase: usize, block: usize) {
work.run_block(phase, block);
}
#[cfg(target_arch = "x86_64")]
#[inline]
#[target_feature(enable = "sse3,ssse3,sse4.1,sse4.2,popcnt,cmpxchg16b")]
fn v2<B: Block>(work: &B, phase: usize, block: usize) {
work.run_block(phase, block);
}
#[cfg(target_arch = "x86_64")]
#[inline]
#[target_feature(
enable = "sse3,ssse3,sse4.1,sse4.2,popcnt,cmpxchg16b,avx,avx2,bmi1,bmi2,f16c,fma,lzcnt,movbe,xsave"
)]
fn v3<B: Block>(work: &B, phase: usize, block: usize) {
work.run_block(phase, block);
}
#[cfg(target_arch = "x86_64")]
#[inline]
#[target_feature(
enable = "sse3,ssse3,sse4.1,sse4.2,popcnt,cmpxchg16b,avx,avx2,bmi1,bmi2,f16c,fma,lzcnt,movbe,xsave,avx512f,avx512bw,avx512cd,avx512dq,avx512vl"
)]
fn v4<B: Block>(work: &B, phase: usize, block: usize) {
work.run_block(phase, block);
}
#[cfg(test)]
mod tests {
use std::sync::atomic::{AtomicUsize, Ordering};
use super::*;
struct Product(AtomicUsize);
impl Block for Product {
#[inline(always)]
fn run_block(&self, phase: usize, block: usize) {
self.0.store(phase * block, Ordering::Relaxed);
}
}
struct Panics;
impl Block for Panics {
fn run_block(&self, _phase: usize, _block: usize) {
panic!("mid-block");
}
}
#[test]
fn a_cap_is_spelled_as_pwrs_spells_it() {
assert_eq!(Level::cap_named("x86-64"), Some(Level::Base));
assert_eq!(Level::cap_named("x86-64-v2"), Some(Level::V2));
assert_eq!(Level::cap_named(" x86-64-v3 "), Some(Level::V3));
assert_eq!(Level::cap_named("x86-64-v4"), Some(Level::V4));
assert_eq!(Level::cap_named("native"), None);
assert_eq!(Level::cap_named(""), None);
assert_eq!(Level::cap_named("x86-64-v5"), None);
}
#[test]
fn the_process_level_is_offered_and_every_offered_level_runs_work() {
let offered = levels_offered();
assert_eq!(offered.first(), Some(&Level::Base));
assert!(offered.contains(&process_level()));
for &level in &offered {
let forced = with_level(level, for_new_job);
assert_eq!(forced, level);
let product = Product(AtomicUsize::new(0));
run_at(level, &product, 6, 7);
assert_eq!(product.0.load(Ordering::Relaxed), 42, "{level:?}");
let sum = AtomicUsize::new(0);
let adds = OneCopy(|phase, block| sum.store(phase + block, Ordering::Relaxed));
run_at(level, &adds, 6, 7);
assert_eq!(sum.load(Ordering::Relaxed), 13, "{level:?}");
}
assert_eq!(
for_new_job(),
process_level(),
"the override ends with with_level"
);
}
#[test]
fn a_panic_in_the_work_reaches_the_caller_at_every_level() {
for level in levels_offered() {
let outcome = std::panic::catch_unwind(|| run_at(level, &Panics, 0, 0));
match outcome {
Ok(()) => panic!("{level:?}: the work's panic did not reach the caller"),
Err(payload) => assert_eq!(payload.downcast_ref::<&str>(), Some(&"mid-block")),
}
}
}
}