Skip to main content

sys_lib/cpu/
features.rs

1//! Instruction-set features, detected through std (which checks operating
2//! system support as well as the CPU, so a feature the OS disabled reads
3//! false), and the x86-64 microarchitecture level.
4
5use core::fmt;
6use std::sync::OnceLock;
7
8macro_rules! features {
9    ($($(#[$doc:meta])* $variant:ident = $name:literal,)*) => {
10        /// An instruction-set feature. [`name`](Feature::name) is rustc's
11        /// target-feature name.
12        #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
13        #[non_exhaustive]
14        pub enum Feature {
15            $($(#[$doc])* $variant,)*
16        }
17
18        impl Feature {
19            const ALL: &[Feature] = &[$(Feature::$variant,)*];
20
21            /// rustc's target-feature name: `"avx2"`, `"sse4.1"`, `"neon"`.
22            ///
23            /// ```
24            /// assert_eq!(sys_lib::cpu::Feature::Sse41.name(), "sse4.1");
25            /// ```
26            #[must_use]
27            pub const fn name(self) -> &'static str {
28                match self {
29                    $(Feature::$variant => $name,)*
30                }
31            }
32        }
33    };
34}
35
36features! {
37    /// x86 SSE2.
38    Sse2 = "sse2",
39    /// x86 SSE3.
40    Sse3 = "sse3",
41    /// x86 SSSE3.
42    Ssse3 = "ssse3",
43    /// x86 SSE4.1.
44    Sse41 = "sse4.1",
45    /// x86 SSE4.2.
46    Sse42 = "sse4.2",
47    /// x86 POPCNT.
48    Popcnt = "popcnt",
49    /// x86 AVX.
50    Avx = "avx",
51    /// x86 AVX2.
52    Avx2 = "avx2",
53    /// x86 FMA3.
54    Fma = "fma",
55    /// x86 F16C.
56    F16c = "f16c",
57    /// x86 BMI1.
58    Bmi1 = "bmi1",
59    /// x86 BMI2.
60    Bmi2 = "bmi2",
61    /// x86 LZCNT.
62    Lzcnt = "lzcnt",
63    /// x86 MOVBE.
64    Movbe = "movbe",
65    /// x86 AVX-512 Foundation.
66    Avx512f = "avx512f",
67    /// x86 AVX-512 byte and word.
68    Avx512bw = "avx512bw",
69    /// x86 AVX-512 conflict detection.
70    Avx512cd = "avx512cd",
71    /// x86 AVX-512 doubleword and quadword.
72    Avx512dq = "avx512dq",
73    /// x86 AVX-512 vector length.
74    Avx512vl = "avx512vl",
75    /// x86 AVX-512 VNNI.
76    Avx512vnni = "avx512vnni",
77    /// x86 AVX-512 VBMI.
78    Avx512vbmi = "avx512vbmi",
79    /// x86 AVX-VNNI.
80    AvxVnni = "avxvnni",
81    /// x86 AES-NI.
82    Aes = "aes",
83    /// x86 carry-less multiplication.
84    Pclmulqdq = "pclmulqdq",
85    /// x86 vector AES.
86    Vaes = "vaes",
87    /// x86 vector carry-less multiplication.
88    Vpclmulqdq = "vpclmulqdq",
89    /// x86 SHA extensions.
90    Sha = "sha",
91    /// x86 RDRAND.
92    Rdrand = "rdrand",
93    /// x86 RDSEED.
94    Rdseed = "rdseed",
95    /// x86 ADX.
96    Adx = "adx",
97    /// x86 GFNI.
98    Gfni = "gfni",
99    /// AArch64 Advanced SIMD.
100    Neon = "neon",
101    /// AArch64 CRC32.
102    Crc32 = "crc",
103    /// AArch64 AES.
104    AesArm = "aes",
105    /// AArch64 polynomial multiply.
106    Pmull = "pmull",
107    /// AArch64 SHA-1 and SHA-256.
108    Sha2 = "sha2",
109    /// AArch64 SHA-3 and SHA-512 (std's `sha3`).
110    Sha3 = "sha3",
111    /// AArch64 dot product.
112    Dotprod = "dotprod",
113    /// AArch64 half-precision arithmetic.
114    Fp16 = "fp16",
115    /// AArch64 large system extensions (atomics).
116    Lse = "lse",
117    /// AArch64 SVE.
118    Sve = "sve",
119    /// AArch64 SVE2.
120    Sve2 = "sve2",
121    /// AArch64 release-consistent loads.
122    Rcpc = "rcpc",
123    /// AArch64 BFloat16.
124    Bf16 = "bf16",
125    /// AArch64 int8 matrix multiply.
126    I8mm = "i8mm",
127}
128
129/// The features detected on this machine, as a set.
130#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Default)]
131pub struct Features {
132    bits: u64,
133}
134
135impl Features {
136    fn bit(feature: Feature) -> u64 {
137        1 << (feature as u32)
138    }
139
140    #[cfg_attr(
141        not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")),
142        allow(dead_code)
143    )]
144    fn insert(&mut self, feature: Feature) {
145        self.bits |= Self::bit(feature);
146    }
147
148    /// Whether `feature` is in the set.
149    #[must_use]
150    pub fn contains(&self, feature: Feature) -> bool {
151        self.bits & Self::bit(feature) != 0
152    }
153
154    /// The features in the set, in declaration order.
155    pub fn iter(&self) -> impl Iterator<Item = Feature> + '_ {
156        Feature::ALL.iter().copied().filter(|f| self.contains(*f))
157    }
158
159    /// How many features are in the set.
160    #[must_use]
161    pub fn len(&self) -> usize {
162        self.bits.count_ones() as usize
163    }
164
165    /// Whether the set is empty (an architecture std detects nothing on).
166    #[must_use]
167    pub fn is_empty(&self) -> bool {
168        self.bits == 0
169    }
170}
171
172impl fmt::Display for Features {
173    /// The names, space-separated, for a log line: `"sse2 sse3 ... avx2 aes"`.
174    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
175        for (i, feature) in self.iter().enumerate() {
176            if i > 0 {
177                f.write_str(" ")?;
178            }
179            f.write_str(feature.name())?;
180        }
181        Ok(())
182    }
183}
184
185#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
186fn detect() -> Features {
187    let mut set = Features::default();
188    macro_rules! check {
189        ($($variant:ident: $name:tt),* $(,)?) => {
190            $(if std::arch::is_x86_feature_detected!($name) { set.insert(Feature::$variant); })*
191        };
192    }
193    check!(
194        Sse2: "sse2", Sse3: "sse3", Ssse3: "ssse3", Sse41: "sse4.1", Sse42: "sse4.2",
195        Popcnt: "popcnt", Avx: "avx", Avx2: "avx2", Fma: "fma", F16c: "f16c",
196        Bmi1: "bmi1", Bmi2: "bmi2", Lzcnt: "lzcnt", Movbe: "movbe",
197        Avx512f: "avx512f", Avx512bw: "avx512bw", Avx512cd: "avx512cd",
198        Avx512dq: "avx512dq", Avx512vl: "avx512vl", Avx512vnni: "avx512vnni",
199        Avx512vbmi: "avx512vbmi", AvxVnni: "avxvnni", Aes: "aes",
200        Pclmulqdq: "pclmulqdq", Vaes: "vaes", Vpclmulqdq: "vpclmulqdq", Sha: "sha",
201        Rdrand: "rdrand", Rdseed: "rdseed", Adx: "adx", Gfni: "gfni",
202    );
203    set
204}
205
206#[cfg(target_arch = "aarch64")]
207fn detect() -> Features {
208    let mut set = Features::default();
209    macro_rules! check {
210        ($($variant:ident: $name:tt),* $(,)?) => {
211            $(if std::arch::is_aarch64_feature_detected!($name) { set.insert(Feature::$variant); })*
212        };
213    }
214    check!(
215        Neon: "neon", Crc32: "crc", AesArm: "aes", Pmull: "pmull", Sha2: "sha2",
216        Sha3: "sha3", Dotprod: "dotprod", Fp16: "fp16", Lse: "lse", Sve: "sve",
217        Sve2: "sve2", Rcpc: "rcpc", Bf16: "bf16", I8mm: "i8mm",
218    );
219    set
220}
221
222#[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]
223fn detect() -> Features {
224    Features::default()
225}
226
227/// The features this machine supports, detected once (about 1 ns afterwards).
228///
229/// ```
230/// println!("features: {}", sys_lib::cpu::features());
231/// ```
232#[must_use]
233pub fn features() -> &'static Features {
234    static FEATURES: OnceLock<Features> = OnceLock::new();
235    FEATURES.get_or_init(detect)
236}
237
238/// Whether this machine supports `feature`: the same answer as std's
239/// `is_x86_feature_detected!` and `is_aarch64_feature_detected!`, which the
240/// compiler's `#[target_feature]` dispatch trusts.
241///
242/// ```
243/// use sys_lib::cpu::{self, Feature};
244///
245/// if cpu::has(Feature::Avx2) {
246///     // pick the AVX2 kernel
247/// }
248/// ```
249#[must_use]
250pub fn has(feature: Feature) -> bool {
251    features().contains(feature)
252}
253
254/// An x86-64 microarchitecture level (the psABI's `x86-64-v1` to `v4`).
255#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
256#[non_exhaustive]
257pub enum X86Level {
258    /// The x86-64 baseline (SSE2).
259    V1,
260    /// Adds SSE3, SSSE3, SSE4.1, SSE4.2, POPCNT, CMPXCHG16B.
261    V2,
262    /// Adds AVX, AVX2, BMI1, BMI2, F16C, FMA, LZCNT, MOVBE, XSAVE.
263    V3,
264    /// Adds AVX-512 F, BW, CD, DQ, VL.
265    V4,
266}
267
268#[cfg(target_arch = "x86_64")]
269fn level() -> Option<X86Level> {
270    use std::arch::is_x86_feature_detected as has;
271    let v2 = has!("sse3")
272        && has!("ssse3")
273        && has!("sse4.1")
274        && has!("sse4.2")
275        && has!("popcnt")
276        && has!("cmpxchg16b");
277    let v3 = v2
278        && has!("avx")
279        && has!("avx2")
280        && has!("bmi1")
281        && has!("bmi2")
282        && has!("f16c")
283        && has!("fma")
284        && has!("lzcnt")
285        && has!("movbe")
286        && has!("xsave");
287    let v4 = v3
288        && has!("avx512f")
289        && has!("avx512bw")
290        && has!("avx512cd")
291        && has!("avx512dq")
292        && has!("avx512vl");
293    Some(match (v2, v3, v4) {
294        (_, _, true) => X86Level::V4,
295        (_, true, _) => X86Level::V3,
296        (true, _, _) => X86Level::V2,
297        _ => X86Level::V1,
298    })
299}
300
301#[cfg(not(target_arch = "x86_64"))]
302fn level() -> Option<X86Level> {
303    None
304}
305
306/// The x86-64 microarchitecture level this machine reaches; `None` off
307/// x86-64. Under Rosetta 2 it is what Rosetta emulates (no AVX: `V2`).
308///
309/// ```
310/// if let Some(level) = sys_lib::cpu::x86_level() { println!("x86-64-{level:?}"); }
311/// ```
312#[must_use]
313pub fn x86_level() -> Option<X86Level> {
314    static LEVEL: OnceLock<Option<X86Level>> = OnceLock::new();
315    *LEVEL.get_or_init(level)
316}
317
318#[cfg(test)]
319mod tests {
320    use super::*;
321
322    #[test]
323    fn test_cpu_features_fit_the_bitset_and_names_are_unique_per_arch() {
324        assert!(Feature::ALL.len() <= 64);
325        let mut x86: Vec<&str> = Feature::ALL[..Feature::ALL
326            .iter()
327            .position(|f| *f == Feature::Neon)
328            .unwrap()]
329            .iter()
330            .map(|f| f.name())
331            .collect();
332        x86.sort_unstable();
333        x86.dedup();
334        assert_eq!(x86.len(), 31);
335    }
336
337    #[test]
338    fn test_cpu_features_agree_with_std_and_the_level() {
339        let set = features();
340        assert_eq!(set.len(), set.iter().count());
341        assert_eq!(
342            set.to_string().split(' ').filter(|s| !s.is_empty()).count(),
343            set.len()
344        );
345        #[cfg(target_arch = "x86_64")]
346        {
347            assert!(has(Feature::Sse2), "SSE2 is the x86-64 baseline");
348            assert_eq!(
349                has(Feature::Avx2),
350                std::arch::is_x86_feature_detected!("avx2")
351            );
352            let level = x86_level().unwrap();
353            assert!(level >= X86Level::V1);
354            if level >= X86Level::V3 {
355                assert!(has(Feature::Avx2) && has(Feature::Fma));
356            }
357        }
358        #[cfg(target_arch = "aarch64")]
359        {
360            assert!(has(Feature::Neon), "NEON is the AArch64 baseline");
361            assert_eq!(x86_level(), None);
362        }
363    }
364}