include!("common.rs");
#[cfg_attr(test, derive(Debug, PartialEq))]
struct AA64Reg {
aa64isar0: u64,
aa64isar1: u64,
#[cfg(test)]
aa64isar3: u64,
aa64mmfr2: u64,
}
#[cold]
fn _detect(info: &mut CpuInfo) {
let AA64Reg {
aa64isar0,
aa64isar1,
#[cfg(test)]
aa64isar3,
aa64mmfr2,
} = imp::aa64reg();
let atomic = extract(aa64isar0, 23, 20);
if atomic >= 0b0010 {
info.set(CpuInfoFlag::lse);
if atomic >= 0b0011 {
info.set(CpuInfoFlag::lse128);
}
}
let lrcpc = extract(aa64isar1, 23, 20);
if lrcpc >= 0b0011 {
info.set(CpuInfoFlag::rcpc3);
}
#[cfg(test)]
if lrcpc >= 0b0001 {
info.set(CpuInfoFlag::rcpc);
if lrcpc >= 0b0010 {
info.set(CpuInfoFlag::rcpc2);
}
}
#[cfg(test)]
if extract(aa64isar3, 19, 16) >= 0b0001 {
info.set(CpuInfoFlag::lsfe);
}
if extract(aa64mmfr2, 35, 32) >= 0b0001 {
info.set(CpuInfoFlag::lse2);
}
}
fn extract(x: u64, high: usize, low: usize) -> u64 {
(x >> low) & ((1 << (high - low + 1)) - 1)
}
#[cfg(not(any(target_os = "netbsd", target_os = "openbsd")))]
mod imp {
#[cfg(not(portable_atomic_no_asm))]
use core::arch::asm;
use super::AA64Reg;
pub(super) fn aa64reg() -> AA64Reg {
unsafe {
let aa64isar0: u64;
asm!(
"mrs {}, ID_AA64ISAR0_EL1",
out(reg) aa64isar0,
options(pure, nomem, nostack, preserves_flags),
);
let aa64isar1: u64;
asm!(
"mrs {}, ID_AA64ISAR1_EL1",
out(reg) aa64isar1,
options(pure, nomem, nostack, preserves_flags),
);
#[cfg(test)]
#[cfg(not(portable_atomic_pre_llvm_18))]
let aa64isar3: u64;
#[cfg(test)]
#[cfg(not(portable_atomic_pre_llvm_18))]
asm!(
"mrs {}, ID_AA64ISAR3_EL1",
out(reg) aa64isar3,
options(pure, nomem, nostack, preserves_flags),
);
let aa64mmfr2: u64;
asm!(
"mrs {}, ID_AA64MMFR2_EL1",
out(reg) aa64mmfr2,
options(pure, nomem, nostack, preserves_flags),
);
AA64Reg {
aa64isar0,
aa64isar1,
#[cfg(test)]
#[cfg(not(portable_atomic_pre_llvm_18))]
aa64isar3,
#[cfg(test)]
#[cfg(portable_atomic_pre_llvm_18)]
aa64isar3: 0,
aa64mmfr2,
}
}
}
}
#[cfg(target_os = "netbsd")]
mod imp {
use core::{mem, ptr};
use super::AA64Reg;
#[allow(non_camel_case_types)]
pub(super) mod ffi {
pub(crate) use crate::utils::ffi::{CStr, c_char, c_int, c_size_t, c_void};
sys_struct!({
pub(crate) struct aarch64_sysctl_cpu_id {
pub(crate) ac_midr: u64,
pub(crate) ac_revidr: u64,
pub(crate) ac_mpidr: u64,
pub(crate) ac_aa64dfr0: u64,
pub(crate) ac_aa64dfr1: u64,
pub(crate) ac_aa64isar0: u64,
pub(crate) ac_aa64isar1: u64,
pub(crate) ac_aa64mmfr0: u64,
pub(crate) ac_aa64mmfr1: u64,
pub(crate) ac_aa64mmfr2: u64,
pub(crate) ac_aa64pfr0: u64,
pub(crate) ac_aa64pfr1: u64,
pub(crate) ac_aa64zfr0: u64,
pub(crate) ac_mvfr0: u32,
pub(crate) ac_mvfr1: u32,
pub(crate) ac_mvfr2: u32,
pub(crate) ac_pad: u32,
pub(crate) ac_clidr: u64,
pub(crate) ac_ctr: u64,
}
});
sys_fn!({
extern "C" {
pub(crate) fn sysctlbyname(
name: *const c_char,
old_p: *mut c_void,
old_len_p: *mut c_size_t,
new_p: *const c_void,
new_len: c_size_t,
) -> c_int;
}
});
}
pub(super) fn sysctl_cpu_id(name: &ffi::CStr) -> Option<AA64Reg> {
const OUT_LEN: ffi::c_size_t =
mem::size_of::<ffi::aarch64_sysctl_cpu_id>() as ffi::c_size_t;
let mut buf: ffi::aarch64_sysctl_cpu_id = unsafe { mem::zeroed() };
let mut out_len = OUT_LEN;
let res = unsafe {
ffi::sysctlbyname(
name.as_ptr(),
(&mut buf as *mut ffi::aarch64_sysctl_cpu_id).cast::<ffi::c_void>(),
&mut out_len,
ptr::null_mut(),
0,
)
};
if res != 0 {
return None;
}
Some(AA64Reg {
aa64isar0: buf.ac_aa64isar0,
aa64isar1: buf.ac_aa64isar1,
#[cfg(test)]
aa64isar3: 0,
aa64mmfr2: buf.ac_aa64mmfr2,
})
}
pub(super) fn aa64reg() -> AA64Reg {
match sysctl_cpu_id(c!("machdep.cpu0.cpu_id")) {
Some(cpu_id) => cpu_id,
None => AA64Reg {
aa64isar0: 0,
aa64isar1: 0,
#[cfg(test)]
aa64isar3: 0,
aa64mmfr2: 0,
},
}
}
}
#[cfg(target_os = "openbsd")]
mod imp {
use core::{mem, ptr};
use super::AA64Reg;
pub(super) mod ffi {
pub(crate) use crate::utils::ffi::{c_int, c_size_t, c_uint, c_void};
sys_const!({
pub(crate) const CTL_MACHDEP: c_int = 7;
pub(crate) const CPU_ID_AA64ISAR0: c_int = 2;
pub(crate) const CPU_ID_AA64ISAR1: c_int = 3;
pub(crate) const CPU_ID_AA64MMFR2: c_int = 7;
});
sys_fn!({
extern "C" {
pub(crate) fn sysctl(
name: *const c_int,
name_len: c_uint,
old_p: *mut c_void,
old_len_p: *mut c_size_t,
new_p: *mut c_void,
new_len: c_size_t,
) -> c_int;
}
});
}
pub(super) fn aa64reg() -> AA64Reg {
let aa64isar0 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64ISAR0]).unwrap_or(0);
let aa64isar1 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64ISAR1]).unwrap_or(0);
let aa64mmfr2 = sysctl64(&[ffi::CTL_MACHDEP, ffi::CPU_ID_AA64MMFR2]).unwrap_or(0);
AA64Reg {
aa64isar0,
aa64isar1,
#[cfg(test)]
aa64isar3: 0,
aa64mmfr2,
}
}
fn sysctl64(mib: &[ffi::c_int]) -> Option<u64> {
const OUT_LEN: ffi::c_size_t = mem::size_of::<u64>() as ffi::c_size_t;
let mut out = 0_u64;
let mut out_len = OUT_LEN;
#[allow(clippy::cast_possible_truncation)]
let mib_len = mib.len() as ffi::c_uint;
let res = unsafe {
ffi::sysctl(
mib.as_ptr(),
mib_len,
(&mut out as *mut u64).cast::<ffi::c_void>(),
&mut out_len,
ptr::null_mut(),
0,
)
};
if res == -1 {
return None;
}
debug_assert_eq!(out_len, OUT_LEN);
Some(out)
}
}
#[allow(
clippy::alloc_instead_of_core,
clippy::std_instead_of_alloc,
clippy::std_instead_of_core,
clippy::undocumented_unsafe_blocks,
clippy::wildcard_imports
)]
#[cfg(test)]
mod tests {
use super::*;
#[test]
#[cfg_attr(portable_atomic_test_detect_false, ignore = "detection disabled")]
fn test_aa64reg() {
let AA64Reg { aa64isar0, aa64isar1, aa64isar3, aa64mmfr2 } = imp::aa64reg();
test_helper::eprintln_nocapture!(
"aa64isar0={},aa64isar1={},aa64isar3={},aa64mmfr2={}",
aa64isar0,
aa64isar1,
aa64isar3,
aa64mmfr2,
);
let atomic = extract(aa64isar0, 23, 20);
if detect().lse() {
if detect().lse128() {
assert_eq!(atomic, 0b0011);
} else {
assert_eq!(atomic, 0b0010);
}
} else {
assert_eq!(atomic, 0b0000);
}
let lrcpc = extract(aa64isar1, 23, 20);
if detect().rcpc() {
if detect().rcpc2() {
if detect().rcpc3() {
assert_eq!(lrcpc, 0b0011);
} else {
assert_eq!(lrcpc, 0b0010);
}
} else {
assert_eq!(lrcpc, 0b0001);
}
} else {
assert_eq!(lrcpc, 0b0000);
}
let lsfe = extract(aa64isar3, 19, 16);
if detect().lsfe() {
assert_eq!(lsfe, 0b0001);
} else {
assert_eq!(lsfe, 0b0000);
}
let at = extract(aa64mmfr2, 35, 32);
if detect().lse2() {
assert_eq!(at, 0b0001);
} else {
assert_eq!(at, 0b0000);
}
}
#[allow(clippy::cast_possible_wrap)]
#[cfg(target_os = "netbsd")]
#[test]
fn test_alternative() {
#[cfg(not(portable_atomic_no_asm))]
use std::arch::asm;
use std::{mem, ptr, vec, vec::Vec};
use test_helper::sys;
use super::imp::ffi;
use crate::utils::{RegISize, RegSize, ffi::*};
fn sysctl_cpu_id_no_libc(name: &[&[u8]]) -> Result<AA64Reg, c_int> {
#[inline]
unsafe fn sysctl(
name: *const c_int,
name_len: c_uint,
old_p: *mut c_void,
old_len_p: *mut c_size_t,
new_p: *const c_void,
new_len: c_size_t,
) -> Result<c_int, c_int> {
let mut n = sys::SYS___sysctl as RegSize;
let arg1 = ptr_reg!(name);
let arg2 = name_len as RegSize;
let arg3 = ptr_reg!(old_p);
let arg4 = ptr_reg!(old_len_p);
let arg5 = ptr_reg!(new_p);
let arg6 = new_len as RegSize;
let r: RegISize;
unsafe {
asm!(
"svc 0", "b.cc 2f",
"mov x17, x0",
"mov x0, #-1",
"2:",
inout("x17") n,
inout("x0") arg1 => r,
inout("x1") arg2 => _,
in("x2") arg3,
in("x3") arg4,
in("x4") arg5,
in("x5") arg6,
options(nostack),
);
}
#[allow(clippy::cast_possible_truncation)]
if r as c_int == -1 { Err(n as c_int) } else { Ok(r as c_int) }
}
fn sysctl_nodes(mib: &mut Vec<i32>) -> Result<Vec<sys::sysctlnode>, i32> {
mib.push(sys::CTL_QUERY);
let mut q_node = sys::sysctlnode {
sysctl_flags: sys::SYSCTL_VERS_1,
..unsafe { mem::zeroed() }
};
let qp = (&mut q_node as *mut sys::sysctlnode).cast::<ffi::c_void>();
let sz = mem::size_of::<sys::sysctlnode>();
let mut olen = 0;
#[allow(clippy::cast_possible_truncation)]
let mib_len = mib.len() as c_uint;
unsafe {
sysctl(mib.as_ptr(), mib_len, ptr::null_mut(), &mut olen, qp, sz)?;
}
let mut nodes = Vec::<sys::sysctlnode>::with_capacity(olen / sz);
let np = nodes.as_mut_ptr().cast::<ffi::c_void>();
unsafe {
sysctl(mib.as_ptr(), mib_len, np, &mut olen, qp, sz)?;
nodes.set_len(olen / sz);
}
mib.pop(); Ok(nodes)
}
fn name_to_mib(parts: &[&[u8]]) -> Result<Vec<i32>, i32> {
let mut mib = vec![];
for (part_no, &part) in parts.iter().enumerate() {
let nodes = sysctl_nodes(&mut mib)?;
for node in nodes {
let mut n = vec![];
for b in node.sysctl_name {
if b != 0 {
n.push(b);
}
}
if n == part {
mib.push(node.sysctl_num);
break;
}
}
if mib.len() != part_no + 1 {
return Err(0);
}
}
Ok(mib)
}
const OUT_LEN: ffi::c_size_t =
mem::size_of::<ffi::aarch64_sysctl_cpu_id>() as ffi::c_size_t;
let mib = name_to_mib(name)?;
let mut buf: ffi::aarch64_sysctl_cpu_id = unsafe { mem::zeroed() };
let mut out_len = OUT_LEN;
#[allow(clippy::cast_possible_truncation)]
let mib_len = mib.len() as c_uint;
unsafe {
sysctl(
mib.as_ptr(),
mib_len,
(&mut buf as *mut ffi::aarch64_sysctl_cpu_id).cast::<ffi::c_void>(),
&mut out_len,
ptr::null_mut(),
0,
)?;
}
Ok(AA64Reg {
aa64isar0: buf.ac_aa64isar0,
aa64isar1: buf.ac_aa64isar1,
aa64isar3: 0,
aa64mmfr2: buf.ac_aa64mmfr2,
})
}
assert_eq!(
imp::sysctl_cpu_id(c!("machdep.cpu0.cpu_id")).unwrap(),
sysctl_cpu_id_no_libc(&[b"machdep", b"cpu0", b"cpu_id"]).unwrap()
);
}
#[cfg(target_os = "openbsd")]
#[test]
fn test_alternative() {
use std::{format, process::Command, string::String};
struct SysctlMachdepOutput(String);
impl SysctlMachdepOutput {
fn new() -> Self {
let output = Command::new("sysctl").arg("machdep").output().unwrap();
assert!(output.status.success());
let stdout = String::from_utf8(output.stdout).unwrap();
Self(stdout)
}
fn field(&self, name: &str) -> Option<u64> {
Some(
self.0
.lines()
.find_map(|s| s.strip_prefix(&format!("{}=", name)))?
.parse()
.unwrap(),
)
}
}
let AA64Reg { aa64isar0, aa64isar1, aa64isar3, aa64mmfr2 } = imp::aa64reg();
let sysctl_output = SysctlMachdepOutput::new();
assert_eq!(aa64isar0, sysctl_output.field("machdep.id_aa64isar0").unwrap_or(0));
assert_eq!(aa64isar1, sysctl_output.field("machdep.id_aa64isar1").unwrap_or(0));
assert_eq!(aa64isar3, sysctl_output.field("machdep.id_aa64isar3").unwrap_or(0));
assert_eq!(aa64mmfr2, sysctl_output.field("machdep.id_aa64mmfr2").unwrap_or(0));
}
}