Skip to main content

sys_lib/cpu/
features.rs

1//! Instruction-set features, detected through std (which checks operating
2//! system support as well as the CPU, so a feature the OS disabled reads
3//! false), and the x86-64 microarchitecture level.
4
5use core::fmt;
6use std::sync::OnceLock;
7
8macro_rules! features {
9    ($($(#[$doc:meta])* $variant:ident = $name:literal,)*) => {
10        /// An instruction-set feature. [`name`](Feature::name) is rustc's
11        /// target-feature name.
12        #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
13        #[non_exhaustive]
14        pub enum Feature {
15            $($(#[$doc])* $variant,)*
16        }
17
18        impl Feature {
19            const ALL: &[Feature] = &[$(Feature::$variant,)*];
20
21            /// rustc's target-feature name: `"avx2"`, `"sse4.1"`, `"neon"`.
22            ///
23            /// ```
24            /// assert_eq!(sys_lib::cpu::Feature::Sse41.name(), "sse4.1");
25            /// ```
26            #[must_use]
27            pub const fn name(self) -> &'static str {
28                match self {
29                    $(Feature::$variant => $name,)*
30                }
31            }
32        }
33    };
34}
35
36features! {
37    /// x86 SSE2.
38    Sse2 = "sse2",
39    /// x86 SSE3.
40    Sse3 = "sse3",
41    /// x86 SSSE3.
42    Ssse3 = "ssse3",
43    /// x86 SSE4.1.
44    Sse41 = "sse4.1",
45    /// x86 SSE4.2.
46    Sse42 = "sse4.2",
47    /// x86 POPCNT.
48    Popcnt = "popcnt",
49    /// x86 AVX.
50    Avx = "avx",
51    /// x86 AVX2.
52    Avx2 = "avx2",
53    /// x86 FMA3.
54    Fma = "fma",
55    /// x86 F16C.
56    F16c = "f16c",
57    /// x86 BMI1.
58    Bmi1 = "bmi1",
59    /// x86 BMI2.
60    Bmi2 = "bmi2",
61    /// x86 LZCNT.
62    Lzcnt = "lzcnt",
63    /// x86 MOVBE.
64    Movbe = "movbe",
65    /// x86 AVX-512 Foundation.
66    Avx512f = "avx512f",
67    /// x86 AVX-512 byte and word.
68    Avx512bw = "avx512bw",
69    /// x86 AVX-512 conflict detection.
70    Avx512cd = "avx512cd",
71    /// x86 AVX-512 doubleword and quadword.
72    Avx512dq = "avx512dq",
73    /// x86 AVX-512 vector length.
74    Avx512vl = "avx512vl",
75    /// x86 AVX-512 VNNI.
76    Avx512vnni = "avx512vnni",
77    /// x86 AVX-512 VBMI.
78    Avx512vbmi = "avx512vbmi",
79    /// x86 AVX-VNNI.
80    AvxVnni = "avxvnni",
81    /// x86 AES-NI.
82    Aes = "aes",
83    /// x86 carry-less multiplication.
84    Pclmulqdq = "pclmulqdq",
85    /// x86 vector AES.
86    Vaes = "vaes",
87    /// x86 vector carry-less multiplication.
88    Vpclmulqdq = "vpclmulqdq",
89    /// x86 SHA extensions.
90    Sha = "sha",
91    /// x86 RDRAND.
92    Rdrand = "rdrand",
93    /// x86 RDSEED.
94    Rdseed = "rdseed",
95    /// x86 ADX.
96    Adx = "adx",
97    /// x86 GFNI.
98    Gfni = "gfni",
99    /// AArch64 Advanced SIMD.
100    Neon = "neon",
101    /// AArch64 CRC32.
102    Crc32 = "crc",
103    /// AArch64 AES.
104    AesArm = "aes",
105    /// AArch64 polynomial multiply.
106    Pmull = "pmull",
107    /// AArch64 SHA-1 and SHA-256.
108    Sha2 = "sha2",
109    /// AArch64 SHA-3 and SHA-512 (std's `sha3`).
110    Sha3 = "sha3",
111    /// AArch64 dot product.
112    Dotprod = "dotprod",
113    /// AArch64 half-precision arithmetic.
114    Fp16 = "fp16",
115    /// AArch64 large system extensions (atomics).
116    Lse = "lse",
117    /// AArch64 SVE.
118    Sve = "sve",
119    /// AArch64 SVE2.
120    Sve2 = "sve2",
121    /// AArch64 release-consistent loads.
122    Rcpc = "rcpc",
123    /// AArch64 BFloat16.
124    Bf16 = "bf16",
125    /// AArch64 int8 matrix multiply.
126    I8mm = "i8mm",
127}
128
129/// The features detected on this machine, as a set.
130#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Default)]
131pub struct Features {
132    bits: u64,
133}
134
135impl Features {
136    fn bit(feature: Feature) -> u64 {
137        1 << (feature as u32)
138    }
139
140    fn insert(&mut self, feature: Feature) {
141        self.bits |= Self::bit(feature);
142    }
143
144    /// Whether `feature` is in the set.
145    #[must_use]
146    pub fn contains(&self, feature: Feature) -> bool {
147        self.bits & Self::bit(feature) != 0
148    }
149
150    /// The features in the set, in declaration order.
151    pub fn iter(&self) -> impl Iterator<Item = Feature> + '_ {
152        Feature::ALL.iter().copied().filter(|f| self.contains(*f))
153    }
154
155    /// How many features are in the set.
156    #[must_use]
157    pub fn len(&self) -> usize {
158        self.bits.count_ones() as usize
159    }
160
161    /// Whether the set is empty (an architecture std detects nothing on).
162    #[must_use]
163    pub fn is_empty(&self) -> bool {
164        self.bits == 0
165    }
166}
167
168impl fmt::Display for Features {
169    /// The names, space-separated, for a log line: `"sse2 sse3 ... avx2 aes"`.
170    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
171        for (i, feature) in self.iter().enumerate() {
172            if i > 0 {
173                f.write_str(" ")?;
174            }
175            f.write_str(feature.name())?;
176        }
177        Ok(())
178    }
179}
180
181#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
182fn detect() -> Features {
183    let mut set = Features::default();
184    macro_rules! check {
185        ($($variant:ident: $name:tt),* $(,)?) => {
186            $(if std::arch::is_x86_feature_detected!($name) { set.insert(Feature::$variant); })*
187        };
188    }
189    check!(
190        Sse2: "sse2", Sse3: "sse3", Ssse3: "ssse3", Sse41: "sse4.1", Sse42: "sse4.2",
191        Popcnt: "popcnt", Avx: "avx", Avx2: "avx2", Fma: "fma", F16c: "f16c",
192        Bmi1: "bmi1", Bmi2: "bmi2", Lzcnt: "lzcnt", Movbe: "movbe",
193        Avx512f: "avx512f", Avx512bw: "avx512bw", Avx512cd: "avx512cd",
194        Avx512dq: "avx512dq", Avx512vl: "avx512vl", Avx512vnni: "avx512vnni",
195        Avx512vbmi: "avx512vbmi", AvxVnni: "avxvnni", Aes: "aes",
196        Pclmulqdq: "pclmulqdq", Vaes: "vaes", Vpclmulqdq: "vpclmulqdq", Sha: "sha",
197        Rdrand: "rdrand", Rdseed: "rdseed", Adx: "adx", Gfni: "gfni",
198    );
199    set
200}
201
202#[cfg(target_arch = "aarch64")]
203fn detect() -> Features {
204    let mut set = Features::default();
205    macro_rules! check {
206        ($($variant:ident: $name:tt),* $(,)?) => {
207            $(if std::arch::is_aarch64_feature_detected!($name) { set.insert(Feature::$variant); })*
208        };
209    }
210    check!(
211        Neon: "neon", Crc32: "crc", AesArm: "aes", Pmull: "pmull", Sha2: "sha2",
212        Sha3: "sha3", Dotprod: "dotprod", Fp16: "fp16", Lse: "lse", Sve: "sve",
213        Sve2: "sve2", Rcpc: "rcpc", Bf16: "bf16", I8mm: "i8mm",
214    );
215    set
216}
217
218#[cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))]
219fn detect() -> Features {
220    Features::default()
221}
222
223/// The features this machine supports, detected once (about 1 ns afterwards).
224///
225/// ```
226/// println!("features: {}", sys_lib::cpu::features());
227/// ```
228#[must_use]
229pub fn features() -> &'static Features {
230    static FEATURES: OnceLock<Features> = OnceLock::new();
231    FEATURES.get_or_init(detect)
232}
233
234/// Whether this machine supports `feature`: the same answer as std's
235/// `is_x86_feature_detected!` and `is_aarch64_feature_detected!`, which the
236/// compiler's `#[target_feature]` dispatch trusts.
237///
238/// ```
239/// use sys_lib::cpu::{self, Feature};
240///
241/// if cpu::has(Feature::Avx2) {
242///     // pick the AVX2 kernel
243/// }
244/// ```
245#[must_use]
246pub fn has(feature: Feature) -> bool {
247    features().contains(feature)
248}
249
250/// An x86-64 microarchitecture level (the psABI's `x86-64-v1` to `v4`).
251#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
252#[non_exhaustive]
253pub enum X86Level {
254    /// The x86-64 baseline (SSE2).
255    V1,
256    /// Adds SSE3, SSSE3, SSE4.1, SSE4.2, POPCNT, CMPXCHG16B.
257    V2,
258    /// Adds AVX, AVX2, BMI1, BMI2, F16C, FMA, LZCNT, MOVBE, XSAVE.
259    V3,
260    /// Adds AVX-512 F, BW, CD, DQ, VL.
261    V4,
262}
263
264#[cfg(target_arch = "x86_64")]
265fn level() -> Option<X86Level> {
266    use std::arch::is_x86_feature_detected as has;
267    let v2 = has!("sse3")
268        && has!("ssse3")
269        && has!("sse4.1")
270        && has!("sse4.2")
271        && has!("popcnt")
272        && has!("cmpxchg16b");
273    let v3 = v2
274        && has!("avx")
275        && has!("avx2")
276        && has!("bmi1")
277        && has!("bmi2")
278        && has!("f16c")
279        && has!("fma")
280        && has!("lzcnt")
281        && has!("movbe")
282        && has!("xsave");
283    let v4 = v3
284        && has!("avx512f")
285        && has!("avx512bw")
286        && has!("avx512cd")
287        && has!("avx512dq")
288        && has!("avx512vl");
289    Some(match (v2, v3, v4) {
290        (_, _, true) => X86Level::V4,
291        (_, true, _) => X86Level::V3,
292        (true, _, _) => X86Level::V2,
293        _ => X86Level::V1,
294    })
295}
296
297#[cfg(not(target_arch = "x86_64"))]
298fn level() -> Option<X86Level> {
299    None
300}
301
302/// The x86-64 microarchitecture level this machine reaches; `None` off
303/// x86-64. Under Rosetta 2 it is what Rosetta emulates (no AVX: `V2`).
304///
305/// ```
306/// if let Some(level) = sys_lib::cpu::x86_level() { println!("x86-64-{level:?}"); }
307/// ```
308#[must_use]
309pub fn x86_level() -> Option<X86Level> {
310    static LEVEL: OnceLock<Option<X86Level>> = OnceLock::new();
311    *LEVEL.get_or_init(level)
312}
313
314#[cfg(test)]
315mod tests {
316    use super::*;
317
318    #[test]
319    fn test_cpu_features_fit_the_bitset_and_names_are_unique_per_arch() {
320        assert!(Feature::ALL.len() <= 64);
321        let mut x86: Vec<&str> = Feature::ALL[..Feature::ALL
322            .iter()
323            .position(|f| *f == Feature::Neon)
324            .unwrap()]
325            .iter()
326            .map(|f| f.name())
327            .collect();
328        x86.sort_unstable();
329        x86.dedup();
330        assert_eq!(x86.len(), 31);
331    }
332
333    #[test]
334    fn test_cpu_features_agree_with_std_and_the_level() {
335        let set = features();
336        assert_eq!(set.len(), set.iter().count());
337        assert_eq!(
338            set.to_string().split(' ').filter(|s| !s.is_empty()).count(),
339            set.len()
340        );
341        #[cfg(target_arch = "x86_64")]
342        {
343            assert!(has(Feature::Sse2), "SSE2 is the x86-64 baseline");
344            assert_eq!(
345                has(Feature::Avx2),
346                std::arch::is_x86_feature_detected!("avx2")
347            );
348            let level = x86_level().unwrap();
349            assert!(level >= X86Level::V1);
350            if level >= X86Level::V3 {
351                assert!(has(Feature::Avx2) && has(Feature::Fma));
352            }
353        }
354        #[cfg(target_arch = "aarch64")]
355        {
356            assert!(has(Feature::Neon), "NEON is the AArch64 baseline");
357            assert_eq!(x86_level(), None);
358        }
359    }
360}