#[cfg(target_arch = "x86_64")]
fn detect_x86_64() -> Detected {
let caps_static = caps_static();
#[cfg(feature = "std")]
let (runtime_caps, is_amd, family, model, amx_permission) = {
let batch = cpuid_batch_x86_64();
(
batch.caps,
batch.is_amd,
batch.family,
batch.model,
batch.amx_permission,
)
};
#[cfg(feature = "std")]
let mut caps = caps_static.union(runtime_caps);
#[cfg(not(feature = "std"))]
let mut caps = caps_static;
#[cfg(feature = "std")]
{
caps = gate_x86_amx_permission(caps, amx_permission);
}
#[cfg(all(
not(feature = "std"),
any(target_os = "linux", target_os = "android")
))]
{
caps = gate_x86_amx_permission(caps, false);
}
#[cfg(feature = "std")]
{
use crate::platform::caps::x86;
if is_intel_hybrid(is_amd, family, model) && !hybrid_avx512_override() {
caps = caps
.difference(x86::AVX512F)
.difference(x86::AVX512DQ)
.difference(x86::AVX512IFMA)
.difference(x86::AVX512CD)
.difference(x86::AVX512BW)
.difference(x86::AVX512VL)
.difference(x86::AVX512VBMI)
.difference(x86::AVX512VBMI2)
.difference(x86::AVX512VNNI)
.difference(x86::AVX512BITALG)
.difference(x86::AVX512VPOPCNTDQ)
.difference(x86::AVX512BF16)
.difference(x86::AVX512FP16)
.difference(x86::AVX512VP2INTERSECT)
.difference(x86::AVX10_1)
.difference(x86::AVX10_2);
}
}
Detected {
caps,
arch: Arch::X86_64,
}
}
#[cfg(target_arch = "x86_64")]
const X86_ALL_AMX: Caps = crate::platform::caps::x86::AMX_TILE
.union(crate::platform::caps::x86::AMX_BF16)
.union(crate::platform::caps::x86::AMX_INT8)
.union(crate::platform::caps::x86::AMX_FP16)
.union(crate::platform::caps::x86::AMX_COMPLEX);
#[cfg(target_arch = "x86_64")]
#[inline]
const fn gate_x86_amx_permission(caps: Caps, permitted: bool) -> Caps {
if permitted {
caps
} else {
caps.difference(X86_ALL_AMX)
}
}
#[cfg(target_arch = "x86")]
fn detect_x86() -> Detected {
let mut caps = caps_static();
#[cfg(feature = "std")]
{
use crate::platform::caps::x86;
if std::arch::is_x86_feature_detected!("sse2") {
caps |= x86::SSE2;
}
caps |= runtime_x86_32();
}
Detected {
caps,
arch: Arch::X86,
}
}
#[cfg(all(target_arch = "x86_64", feature = "std"))]
#[inline]
#[allow(unsafe_code, unused_unsafe)]
fn cpuid_leaf(leaf: u32) -> core::arch::x86_64::CpuidResult {
unsafe { core::arch::x86_64::__cpuid(leaf) }
}
#[cfg(all(target_arch = "x86_64", feature = "std"))]
#[inline]
#[allow(unsafe_code, unused_unsafe)]
fn cpuid_leaf_count(leaf: u32, subleaf: u32) -> core::arch::x86_64::CpuidResult {
unsafe { core::arch::x86_64::__cpuid_count(leaf, subleaf) }
}
#[cfg(all(target_arch = "x86", feature = "std"))]
#[inline]
#[allow(unsafe_code, unused_unsafe)]
fn cpuid_leaf(leaf: u32) -> core::arch::x86::CpuidResult {
unsafe { core::arch::x86::__cpuid(leaf) }
}
#[cfg(all(target_arch = "x86_64", feature = "std"))]
struct CpuidBatch {
caps: Caps,
is_amd: bool,
family: u32,
model: u32,
amx_permission: bool,
}
#[cfg(all(target_arch = "x86_64", feature = "std"))]
#[derive(Clone, Copy, Default)]
struct CpuidRegisters {
eax: u32,
ebx: u32,
ecx: u32,
edx: u32,
}
#[cfg(all(target_arch = "x86_64", feature = "std"))]
impl From<core::arch::x86_64::CpuidResult> for CpuidRegisters {
fn from(result: core::arch::x86_64::CpuidResult) -> Self {
Self {
eax: result.eax,
ebx: result.ebx,
ecx: result.ecx,
edx: result.edx,
}
}
}
#[cfg(all(target_arch = "x86_64", feature = "std"))]
#[derive(Clone, Copy, Default)]
struct CpuidSnapshot {
leaf0: CpuidRegisters,
leaf1: CpuidRegisters,
leaf7_0: CpuidRegisters,
leaf7_1: CpuidRegisters,
leaf24_0: CpuidRegisters,
extended_leaf0: CpuidRegisters,
extended_leaf1: CpuidRegisters,
xcr0: u64,
amx_permission: bool,
}
#[cfg(all(target_arch = "x86_64", feature = "std"))]
#[allow(unsafe_code)]
fn cpuid_batch_x86_64() -> CpuidBatch {
use core::arch::x86_64::_xgetbv;
let leaf0 = CpuidRegisters::from(cpuid_leaf(0));
let leaf1 = if leaf0.eax >= 1 {
CpuidRegisters::from(cpuid_leaf(1))
} else {
CpuidRegisters::default()
};
let leaf7_0 = if leaf0.eax >= 7 {
CpuidRegisters::from(cpuid_leaf_count(7, 0))
} else {
CpuidRegisters::default()
};
let leaf7_1 = if leaf0.eax >= 7 && leaf7_0.eax >= 1 {
CpuidRegisters::from(cpuid_leaf_count(7, 1))
} else {
CpuidRegisters::default()
};
let leaf24_0 = if leaf0.eax >= 0x24 && leaf7_1.edx & (1 << 19) != 0 {
CpuidRegisters::from(cpuid_leaf_count(0x24, 0))
} else {
CpuidRegisters::default()
};
let extended_leaf0 = CpuidRegisters::from(cpuid_leaf(0x8000_0000));
let extended_leaf1 = if extended_leaf0.eax >= 0x8000_0001 {
CpuidRegisters::from(cpuid_leaf(0x8000_0001))
} else {
CpuidRegisters::default()
};
let xcr0 = if leaf1.ecx & (1 << 27) != 0 {
unsafe { _xgetbv(0) }
} else {
0
};
decode_cpuid_x86_64(CpuidSnapshot {
leaf0,
leaf1,
leaf7_0,
leaf7_1,
leaf24_0,
extended_leaf0,
extended_leaf1,
xcr0,
amx_permission: amx_xstate_permission_x86_64(),
})
}
#[cfg(all(
target_arch = "x86_64",
feature = "std",
any(target_os = "linux", target_os = "android")
))]
#[allow(unsafe_code)]
fn amx_xstate_permission_x86_64() -> bool {
const SYS_ARCH_PRCTL: isize = 158;
const ARCH_GET_XCOMP_PERM: usize = 0x1022;
const XCOMP_TILE_MASK: u64 = (1 << 17) | (1 << 18);
let mut permissions = 0u64;
let mut result = SYS_ARCH_PRCTL;
unsafe {
core::arch::asm!(
"syscall",
inlateout("rax") result,
in("rdi") ARCH_GET_XCOMP_PERM,
in("rsi") &mut permissions,
lateout("rcx") _,
lateout("r11") _,
options(nostack),
);
}
result == 0 && permissions & XCOMP_TILE_MASK == XCOMP_TILE_MASK
}
#[cfg(all(
target_arch = "x86_64",
feature = "std",
not(any(target_os = "linux", target_os = "android"))
))]
const fn amx_xstate_permission_x86_64() -> bool {
true
}
#[cfg(all(target_arch = "x86_64", feature = "std"))]
fn decode_cpuid_x86_64(snapshot: CpuidSnapshot) -> CpuidBatch {
use crate::platform::caps::x86;
const XCR0_AVX_MASK: u64 = 0x6;
const XCR0_AVX512_MASK: u64 = 0xE0;
const XCR0_AMX_MASK: u64 = (1 << 17) | (1 << 18);
const XCR0_APX_MASK: u64 = 1 << 19;
let mut caps = Caps::NONE;
let cpuid0 = snapshot.leaf0;
let cpuid1 = if cpuid0.eax >= 1 {
snapshot.leaf1
} else {
CpuidRegisters::default()
};
let cpuid7 = if cpuid0.eax >= 7 {
snapshot.leaf7_0
} else {
CpuidRegisters::default()
};
let cpuid7_1 = if cpuid0.eax >= 7 && cpuid7.eax >= 1 {
snapshot.leaf7_1
} else {
CpuidRegisters::default()
};
let cpuid24 = if cpuid0.eax >= 0x24 {
snapshot.leaf24_0
} else {
CpuidRegisters::default()
};
let cpuid_ext = if snapshot.extended_leaf0.eax >= 0x8000_0001 {
snapshot.extended_leaf1
} else {
CpuidRegisters::default()
};
let is_intel = cpuid0.ebx == 0x756e_6547 && cpuid0.edx == 0x4965_6e69 && cpuid0.ecx == 0x6c65_746e;
let is_amd = cpuid0.ebx == 0x6874_7541;
let base_family = (cpuid1.eax >> 8) & 0xF;
let ext_family = (cpuid1.eax >> 20) & 0xFF;
let family = base_family + ext_family;
let base_model = (cpuid1.eax >> 4) & 0xF;
let ext_model = (cpuid1.eax >> 16) & 0xF;
let model = if base_family == 6 || base_family == 15 {
base_model + (ext_model << 4)
} else {
base_model
};
let osxsave = cpuid1.ecx & (1 << 27) != 0;
let os_avx = osxsave && (snapshot.xcr0 & XCR0_AVX_MASK) == XCR0_AVX_MASK;
let os_avx512 = os_avx && (snapshot.xcr0 & XCR0_AVX512_MASK) == XCR0_AVX512_MASK;
let os_amx =
osxsave && (snapshot.xcr0 & XCR0_AMX_MASK) == XCR0_AMX_MASK && snapshot.amx_permission;
let os_apx = osxsave && (snapshot.xcr0 & XCR0_APX_MASK) == XCR0_APX_MASK;
let has_avx = cpuid1.ecx & (1 << 28) != 0;
let has_avx2 = cpuid7.ebx & (1 << 5) != 0;
let has_avx512f = cpuid7.ebx & (1 << 16) != 0;
let has_avx512bw = cpuid7.ebx & (1 << 30) != 0;
let has_fma = cpuid1.ecx & (1 << 12) != 0;
let has_f16c = cpuid1.ecx & (1 << 29) != 0;
let has_aes = cpuid1.ecx & (1 << 25) != 0;
let has_pclmul = cpuid1.ecx & (1 << 1) != 0;
let rust_avx = os_avx && has_avx;
let rust_avx512 = os_avx512 && rust_avx && has_avx512f && has_fma && has_f16c;
if cpuid1.ecx & (1 << 0) != 0 {
caps |= x86::SSE3;
}
if cpuid1.ecx & (1 << 9) != 0 {
caps |= x86::SSSE3;
}
if cpuid1.ecx & (1 << 19) != 0 {
caps |= x86::SSE41;
}
if cpuid1.ecx & (1 << 20) != 0 {
caps |= x86::SSE42;
}
if cpuid1.ecx & (1 << 23) != 0 {
caps |= x86::POPCNT;
}
if has_aes {
caps |= x86::AESNI;
}
if has_pclmul {
caps |= x86::PCLMULQDQ;
}
if cpuid1.ecx & (1 << 30) != 0 {
caps |= x86::RDRAND;
}
if rust_avx {
caps |= x86::AVX;
if has_fma {
caps |= x86::FMA;
}
if has_f16c {
caps |= x86::F16C;
}
}
if cpuid7.ebx & (1 << 3) != 0 {
caps |= x86::BMI1;
}
if cpuid7.ebx & (1 << 8) != 0 {
caps |= x86::BMI2;
}
if cpuid7.ebx & (1 << 19) != 0 {
caps |= x86::ADX;
}
if cpuid7.ebx & (1 << 29) != 0 {
caps |= x86::SHA;
}
if rust_avx && has_avx2 {
caps |= x86::AVX2;
}
if rust_avx512 {
caps |= x86::AVX512F;
if cpuid7.ebx & (1 << 17) != 0 {
caps |= x86::AVX512DQ;
}
if cpuid7.ebx & (1 << 21) != 0 {
caps |= x86::AVX512IFMA;
}
if cpuid7.ebx & (1 << 28) != 0 {
caps |= x86::AVX512CD;
}
if cpuid7.ebx & (1 << 30) != 0 {
caps |= x86::AVX512BW;
}
if cpuid7.ebx & (1 << 31) != 0 {
caps |= x86::AVX512VL;
}
if cpuid7.ecx & (1 << 1) != 0 {
caps |= x86::AVX512VBMI;
}
if cpuid7.ecx & (1 << 6) != 0 {
caps |= x86::AVX512VBMI2;
}
if cpuid7.ecx & (1 << 11) != 0 {
caps |= x86::AVX512VNNI;
}
if cpuid7.ecx & (1 << 12) != 0 {
caps |= x86::AVX512BITALG;
}
if cpuid7.ecx & (1 << 14) != 0 {
caps |= x86::AVX512VPOPCNTDQ;
}
if cpuid7.edx & (1 << 8) != 0 {
caps |= x86::AVX512VP2INTERSECT;
}
if has_avx512bw && cpuid7.edx & (1 << 23) != 0 {
caps |= x86::AVX512FP16;
}
if has_avx512bw && cpuid7_1.eax & (1 << 5) != 0 {
caps |= x86::AVX512BF16;
}
if cpuid7_1.edx & (1 << 19) != 0 {
let avx10_version = cpuid24.ebx & 0xFF;
if avx10_version >= 1 {
caps |= x86::AVX10_1;
}
}
}
if cpuid7.ecx & (1 << 8) != 0 {
caps |= x86::GFNI;
}
if rust_avx && has_avx2 && has_aes && cpuid7.ecx & (1 << 9) != 0 {
caps |= x86::VAES;
}
if rust_avx && has_pclmul && cpuid7.ecx & (1 << 10) != 0 {
caps |= x86::VPCLMULQDQ;
}
if cpuid7.ebx & (1 << 18) != 0 {
caps |= x86::RDSEED;
}
if cpuid7.ecx & (1 << 27) != 0 {
caps |= x86::MOVDIRI;
}
if cpuid7.ecx & (1 << 28) != 0 {
caps |= x86::MOVDIR64B;
}
if cpuid7.edx & (1 << 14) != 0 {
caps |= x86::SERIALIZE;
}
if os_amx && cpuid7.edx & (1 << 24) != 0 {
caps |= x86::AMX_TILE;
}
if os_amx && cpuid7.edx & (1 << 22) != 0 {
caps |= x86::AMX_BF16;
}
if os_amx && cpuid7.edx & (1 << 25) != 0 {
caps |= x86::AMX_INT8;
}
if cpuid7_1.eax & (1 << 0) != 0 {
caps |= x86::SHA512;
}
if os_amx && cpuid7_1.eax & (1 << 21) != 0 {
caps |= x86::AMX_FP16;
}
if os_amx && cpuid7_1.edx & (1 << 8) != 0 {
caps |= x86::AMX_COMPLEX;
}
if os_apx && cpuid7_1.edx & (1 << 21) != 0 {
caps |= x86::APX;
}
if cpuid_ext.ecx & (1 << 5) != 0 {
caps |= x86::LZCNT;
}
if cpuid_ext.ecx & (1 << 6) != 0 {
caps |= x86::SSE4A;
}
if is_amd {
caps |= x86::AMD;
if family >= 0x1A {
caps |= x86::AMD_ZEN5;
}
}
if is_intel_sapphire_rapids(is_intel, family, model) {
caps |= x86::INTEL_SAPPHIRE_RAPIDS;
}
CpuidBatch {
caps,
is_amd,
family,
model,
amx_permission: snapshot.amx_permission,
}
}
#[cfg(all(target_arch = "x86", feature = "std"))]
#[allow(unsafe_code)]
fn runtime_x86_32() -> Caps {
use crate::platform::caps::x86;
let mut caps = Caps::NONE;
let cpuid1 = cpuid_leaf(1);
if cpuid1.ecx & (1 << 0) != 0 {
caps |= x86::SSE3;
}
if cpuid1.ecx & (1 << 9) != 0 {
caps |= x86::SSSE3;
}
if cpuid1.ecx & (1 << 19) != 0 {
caps |= x86::SSE41;
}
if cpuid1.ecx & (1 << 20) != 0 {
caps |= x86::SSE42;
}
if cpuid1.ecx & (1 << 1) != 0 {
caps |= x86::PCLMULQDQ;
}
if cpuid1.ecx & (1 << 25) != 0 {
caps |= x86::AESNI;
}
caps
}
#[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "std"))]
fn hybrid_avx512_override() -> bool {
let value = std::env::var("RSCRYPTO_FORCE_AVX512").ok();
parse_hybrid_avx512_override(value.as_deref())
}
#[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "std"))]
fn parse_hybrid_avx512_override(value: Option<&str>) -> bool {
matches!(value, Some("1")) || value.is_some_and(|value| value.eq_ignore_ascii_case("true"))
}
#[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "std"))]
fn is_intel_sapphire_rapids(is_intel: bool, family: u32, model: u32) -> bool {
is_intel && family == 6 && model == 0x8F
}
#[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "std"))]
fn is_intel_hybrid(is_amd: bool, family: u32, model: u32) -> bool {
if is_amd {
return false;
}
if family != 6 {
return false;
}
matches!(
model,
0x97 | 0x9A | 0x9C | 0xB7 | 0xBA | 0xBF | 0xAA | 0xAC | 0xBD | 0xC5 | 0xC6 )
}