include!("common.rs");
#[cfg_attr(test, derive(Debug, PartialEq))]
struct AA64Reg {
aa64isar0: u64,
#[cfg(test)]
aa64isar1: u64,
#[cfg_attr(target_os = "openbsd", cfg(test))]
aa64mmfr2: u64,
}
#[cold]
fn _detect(info: &mut CpuInfo) {
let AA64Reg {
aa64isar0,
#[cfg(test)]
aa64isar1,
#[cfg_attr(target_os = "openbsd", cfg(test))]
aa64mmfr2,
} = imp::aa64reg();
let atomic = extract(aa64isar0, 23, 20);
if atomic >= 2 {
info.set(CpuInfo::HAS_LSE);
#[cfg(test)]
{
if atomic >= 3 {
info.set(CpuInfo::HAS_LSE128);
}
}
}
#[cfg(test)]
{
if extract(aa64isar1, 23, 20) >= 3 {
info.set(CpuInfo::HAS_RCPC3);
}
}
#[cfg_attr(target_os = "openbsd", cfg(test))]
{
if extract(aa64mmfr2, 35, 32) >= 1 {
info.set(CpuInfo::HAS_LSE2);
}
}
}
fn extract(x: u64, high: usize, low: usize) -> u64 {
(x >> low) & ((1 << (high - low + 1)) - 1)
}
#[cfg(not(any(target_os = "netbsd", target_os = "openbsd")))]
mod imp {
#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
use super::AA64Reg;
pub(super) fn aa64reg() -> AA64Reg {
unsafe {
let aa64isar0: u64;
asm!(
"mrs {0}, ID_AA64ISAR0_EL1",
out(reg) aa64isar0,
options(pure, nomem, nostack, preserves_flags)
);
#[cfg(test)]
let aa64isar1: u64;
#[cfg(test)]
{
asm!(
"mrs {0}, ID_AA64ISAR1_EL1",
out(reg) aa64isar1,
options(pure, nomem, nostack, preserves_flags)
);
}
let aa64mmfr2: u64;
asm!(
"mrs {0}, ID_AA64MMFR2_EL1",
out(reg) aa64mmfr2,
options(pure, nomem, nostack, preserves_flags)
);
AA64Reg {
aa64isar0,
#[cfg(test)]
aa64isar1,
aa64mmfr2,
}
}
}
}
#[cfg(target_os = "netbsd")]
mod imp {
use core::ptr;
use super::AA64Reg;
#[allow(non_camel_case_types)]
pub(super) mod ffi {
pub(crate) use super::super::c_types::{c_char, c_int, c_size_t, c_void};
extern "C" {
pub(crate) fn sysctlbyname(
name: *const c_char,
old_p: *mut c_void,
old_len_p: *mut c_size_t,
new_p: *const c_void,
new_len: c_size_t,
) -> c_int;
}
#[derive(Clone, Copy)]
#[repr(C)]
pub(crate) struct aarch64_sysctl_cpu_id {
pub(crate) midr: u64,
pub(crate) revidr: u64,
pub(crate) mpidr: u64,
pub(crate) aa64dfr0: u64,
pub(crate) aa64dfr1: u64,
pub(crate) aa64isar0: u64,
pub(crate) aa64isar1: u64,
pub(crate) aa64mmfr0: u64,
pub(crate) aa64mmfr1: u64,
pub(crate) aa64mmfr2: u64,
pub(crate) aa64pfr0: u64,
pub(crate) aa64pfr1: u64,
pub(crate) aa64zfr0: u64,
pub(crate) mvfr0: u32,
pub(crate) mvfr1: u32,
pub(crate) mvfr2: u32,
pub(crate) pad: u32,
pub(crate) clidr: u64,
pub(crate) ctr: u64,
}
}
pub(super) unsafe fn sysctl_cpu_id(name: &[u8]) -> Option<AA64Reg> {
const OUT_LEN: ffi::c_size_t =
core::mem::size_of::<ffi::aarch64_sysctl_cpu_id>() as ffi::c_size_t;
debug_assert_eq!(name.last(), Some(&0), "{:?}", name);
debug_assert_eq!(name.iter().filter(|&&v| v == 0).count(), 1, "{:?}", name);
let mut buf: ffi::aarch64_sysctl_cpu_id = unsafe { core::mem::zeroed() };
let mut out_len = OUT_LEN;
let res = unsafe {
ffi::sysctlbyname(
name.as_ptr().cast::<ffi::c_char>(),
(&mut buf as *mut ffi::aarch64_sysctl_cpu_id).cast::<ffi::c_void>(),
&mut out_len,
ptr::null_mut(),
0,
)
};
if res != 0 {
return None;
}
Some(AA64Reg {
aa64isar0: buf.aa64isar0,
#[cfg(test)]
aa64isar1: buf.aa64isar1,
aa64mmfr2: buf.aa64mmfr2,
})
}
pub(super) fn aa64reg() -> AA64Reg {
match unsafe { sysctl_cpu_id(b"machdep.cpu0.cpu_id\0") } {
Some(cpu_id) => cpu_id,
None => AA64Reg {
aa64isar0: 0,
#[cfg(test)]
aa64isar1: 0,
aa64mmfr2: 0,
},
}
}
}
#[cfg(target_os = "openbsd")]
mod imp {
use core::ptr;
use super::AA64Reg;
#[allow(non_camel_case_types)]
pub(super) mod ffi {
pub(crate) use super::super::c_types::{c_int, c_size_t, c_uint, c_void};
pub(crate) const CTL_MACHDEP: c_int = 7;
pub(crate) const CPU_ID_AA64ISAR0: c_int = 2;
#[cfg(test)]
pub(crate) const CPU_ID_AA64ISAR1: c_int = 3;
#[cfg(test)]
pub(crate) const CPU_ID_AA64MMFR2: c_int = 7;
extern "C" {
pub(crate) fn sysctl(
name: *const c_int,
name_len: c_uint,
old_p: *mut c_void,
old_len_p: *mut c_size_t,
new_p: *mut c_void,
new_len: c_size_t,
) -> c_int;
}
}
pub(super) fn aa64reg() -> AA64Reg {
let aa64isar0 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64ISAR0]).unwrap_or(0);
#[cfg(test)]
let aa64isar1 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64ISAR1]).unwrap_or(0);
#[cfg(test)]
let aa64mmfr2 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64MMFR2]).unwrap_or(0);
AA64Reg {
aa64isar0,
#[cfg(test)]
aa64isar1,
#[cfg(test)]
aa64mmfr2,
}
}
fn sysctl64(mib: &[ffi::c_int]) -> Option<u64> {
const OUT_LEN: ffi::c_size_t = core::mem::size_of::<u64>() as ffi::c_size_t;
let mut out = 0_u64;
let mut out_len = OUT_LEN;
#[allow(clippy::cast_possible_truncation)]
let res = unsafe {
ffi::sysctl(
mib.as_ptr(),
mib.len() as ffi::c_uint,
(&mut out as *mut u64).cast::<ffi::c_void>(),
&mut out_len,
ptr::null_mut(),
0,
)
};
if res == -1 {
return None;
}
debug_assert_eq!(out_len, OUT_LEN);
Some(out)
}
}
#[allow(
clippy::alloc_instead_of_core,
clippy::std_instead_of_alloc,
clippy::std_instead_of_core,
clippy::undocumented_unsafe_blocks,
clippy::wildcard_imports
)]
#[cfg(test)]
mod tests {
use std::{
process::Command,
string::{String, ToString},
};
use super::*;
#[test]
fn test_aa64reg() {
let AA64Reg { aa64isar0, aa64isar1, aa64mmfr2 } = imp::aa64reg();
std::eprintln!("aa64isar0={}", aa64isar0);
std::eprintln!("aa64isar1={}", aa64isar1);
std::eprintln!("aa64mmfr2={}", aa64mmfr2);
if cfg!(target_os = "openbsd") {
let output = Command::new("sysctl").arg("machdep").output().unwrap();
assert!(output.status.success());
let stdout = String::from_utf8(output.stdout).unwrap();
assert_eq!(
stdout.lines().find_map(|s| s.strip_prefix("machdep.id_aa64isar0=")).unwrap_or("0"),
aa64isar0.to_string(),
);
assert_eq!(
stdout.lines().find_map(|s| s.strip_prefix("machdep.id_aa64isar1=")).unwrap_or("0"),
aa64isar1.to_string(),
);
assert_eq!(
stdout.lines().find_map(|s| s.strip_prefix("machdep.id_aa64mmfr2=")).unwrap_or("0"),
aa64mmfr2.to_string(),
);
}
if detect().test(CpuInfo::HAS_LSE) {
let atomic = extract(aa64isar0, 23, 20);
if detect().test(CpuInfo::HAS_LSE128) {
assert_eq!(atomic, 3);
} else {
assert_eq!(atomic, 2);
}
}
if detect().test(CpuInfo::HAS_LSE2) {
assert_eq!(extract(aa64mmfr2, 35, 32), 1);
}
if detect().test(CpuInfo::HAS_RCPC3) {
assert_eq!(extract(aa64isar1, 23, 20), 3);
}
}
#[allow(clippy::cast_possible_wrap)]
#[cfg(target_os = "netbsd")]
#[test]
fn test_netbsd() {
use c_types::*;
use core::{arch::asm, mem, ptr};
use imp::ffi;
use test_helper::sys;
unsafe fn sysctl_cpu_id_asm_syscall(name: &[&[u8]]) -> Result<AA64Reg, c_int> {
#[inline]
unsafe fn sysctl(
name: *const c_int,
name_len: c_uint,
old_p: *mut c_void,
old_len_p: *mut c_size_t,
new_p: *const c_void,
new_len: c_size_t,
) -> Result<c_int, c_int> {
#[allow(clippy::cast_possible_truncation)]
unsafe {
let mut n = sys::SYS___sysctl as u64;
let r: i64;
asm!(
"svc 0",
"b.cc 2f",
"mov x17, x0",
"mov x0, #-1",
"2:",
inout("x17") n,
inout("x0") ptr_reg!(name) => r,
inout("x1") name_len as u64 => _,
in("x2") ptr_reg!(old_p),
in("x3") ptr_reg!(old_len_p),
in("x4") ptr_reg!(new_p),
in("x5") new_len as u64,
options(nostack),
);
if r as c_int == -1 {
Err(n as c_int)
} else {
Ok(r as c_int)
}
}
}
use std::{vec, vec::Vec};
fn sysctl_nodes(mib: &mut Vec<i32>) -> Result<Vec<sys::sysctlnode>, i32> {
mib.push(sys::CTL_QUERY);
let mut q_node = sys::sysctlnode {
sysctl_flags: sys::SYSCTL_VERS_1,
..unsafe { mem::zeroed() }
};
let qp = (&mut q_node as *mut sys::sysctlnode).cast::<ffi::c_void>();
let sz = mem::size_of::<sys::sysctlnode>();
let mut olen = 0;
#[allow(clippy::cast_possible_truncation)]
unsafe {
sysctl(mib.as_ptr(), mib.len() as c_uint, ptr::null_mut(), &mut olen, qp, sz)?;
}
let mut nodes = Vec::<sys::sysctlnode>::with_capacity(olen / sz);
let np = nodes.as_mut_ptr().cast::<ffi::c_void>();
#[allow(clippy::cast_possible_truncation)]
unsafe {
sysctl(mib.as_ptr(), mib.len() as c_uint, np, &mut olen, qp, sz)?;
nodes.set_len(olen / sz);
}
mib.pop(); Ok(nodes)
}
fn name_to_mib(parts: &[&[u8]]) -> Result<Vec<i32>, i32> {
let mut mib = vec![];
for (part_no, &part) in parts.iter().enumerate() {
let nodes = sysctl_nodes(&mut mib)?;
for node in nodes {
let mut n = vec![];
for b in node.sysctl_name {
if b != 0 {
n.push(b);
}
}
if n == part {
mib.push(node.sysctl_num);
break;
}
}
if mib.len() != part_no + 1 {
return Err(0);
}
}
Ok(mib)
}
const OUT_LEN: ffi::c_size_t =
core::mem::size_of::<ffi::aarch64_sysctl_cpu_id>() as ffi::c_size_t;
let mib = name_to_mib(name)?;
let mut buf: ffi::aarch64_sysctl_cpu_id = unsafe { core::mem::zeroed() };
let mut out_len = OUT_LEN;
#[allow(clippy::cast_possible_truncation)]
unsafe {
sysctl(
mib.as_ptr(),
mib.len() as c_uint,
(&mut buf as *mut ffi::aarch64_sysctl_cpu_id).cast::<ffi::c_void>(),
&mut out_len,
ptr::null_mut(),
0,
)?;
}
Ok(AA64Reg {
aa64isar0: buf.aa64isar0,
#[cfg(test)]
aa64isar1: buf.aa64isar1,
#[cfg(test)]
aa64mmfr2: buf.aa64mmfr2,
})
}
unsafe {
assert_eq!(
imp::sysctl_cpu_id(b"machdep.cpu0.cpu_id\0").unwrap(),
sysctl_cpu_id_asm_syscall(&[b"machdep", b"cpu0", b"cpu_id"]).unwrap()
);
}
}
#[cfg(target_os = "netbsd")]
#[allow(
clippy::cast_possible_wrap,
clippy::cast_sign_loss,
clippy::no_effect_underscore_binding,
clippy::used_underscore_binding
)]
const _: fn() = || {
use core::mem::size_of;
use imp::ffi;
use test_helper::{libc, sys};
let mut _sysctlbyname: unsafe extern "C" fn(
*const ffi::c_char,
*mut ffi::c_void,
*mut ffi::c_size_t,
*const ffi::c_void,
ffi::c_size_t,
) -> ffi::c_int = ffi::sysctlbyname;
_sysctlbyname = libc::sysctlbyname;
_sysctlbyname = sys::sysctlbyname;
static_assert!(
size_of::<ffi::aarch64_sysctl_cpu_id>() == size_of::<sys::aarch64_sysctl_cpu_id>()
);
let ffi: ffi::aarch64_sysctl_cpu_id = unsafe { core::mem::zeroed() };
let _ = sys::aarch64_sysctl_cpu_id {
ac_midr: ffi.midr,
ac_revidr: ffi.revidr,
ac_mpidr: ffi.mpidr,
ac_aa64dfr0: ffi.aa64dfr0,
ac_aa64dfr1: ffi.aa64dfr1,
ac_aa64isar0: ffi.aa64isar0,
ac_aa64isar1: ffi.aa64isar1,
ac_aa64mmfr0: ffi.aa64mmfr0,
ac_aa64mmfr1: ffi.aa64mmfr1,
ac_aa64mmfr2: ffi.aa64mmfr2,
ac_aa64pfr0: ffi.aa64pfr0,
ac_aa64pfr1: ffi.aa64pfr1,
ac_aa64zfr0: ffi.aa64zfr0,
ac_mvfr0: ffi.mvfr0,
ac_mvfr1: ffi.mvfr1,
ac_mvfr2: ffi.mvfr2,
ac_pad: ffi.pad,
ac_clidr: ffi.clidr,
ac_ctr: ffi.ctr,
};
};
#[cfg(target_os = "openbsd")]
#[allow(
clippy::cast_possible_wrap,
clippy::cast_sign_loss,
clippy::no_effect_underscore_binding
)]
const _: fn() = || {
use imp::ffi;
use test_helper::{libc, sys};
let mut _sysctl: unsafe extern "C" fn(
*const ffi::c_int,
ffi::c_uint,
*mut ffi::c_void,
*mut ffi::c_size_t,
*mut ffi::c_void,
ffi::c_size_t,
) -> ffi::c_int = ffi::sysctl;
_sysctl = libc::sysctl;
_sysctl = sys::sysctl;
static_assert!(ffi::CTL_MACHDEP == libc::CTL_MACHDEP);
static_assert!(ffi::CTL_MACHDEP == sys::CTL_MACHDEP as ffi::c_int);
static_assert!(ffi::CPU_ID_AA64ISAR0 == sys::CPU_ID_AA64ISAR0 as ffi::c_int);
static_assert!(ffi::CPU_ID_AA64ISAR1 == sys::CPU_ID_AA64ISAR1 as ffi::c_int);
static_assert!(ffi::CPU_ID_AA64MMFR2 == sys::CPU_ID_AA64MMFR2 as ffi::c_int);
};
}