include!("common.rs");
use os::ffi;
#[cfg(any(target_os = "linux", target_os = "android"))]
mod os {
#[cfg_attr(test, allow(dead_code))]
pub(super) mod ffi {
pub(crate) use super::super::c_types::c_ulong;
#[cfg(all(target_arch = "aarch64", target_os = "android"))]
pub(crate) use super::super::c_types::{c_char, c_int};
extern "C" {
pub(crate) fn getauxval(type_: c_ulong) -> c_ulong;
#[cfg(all(target_arch = "aarch64", target_os = "android"))]
pub(crate) fn __system_property_get(name: *const c_char, value: *mut c_char) -> c_int;
}
#[cfg(any(test, target_arch = "aarch64"))]
pub(crate) const AT_HWCAP: c_ulong = 16;
#[cfg(any(test, target_arch = "powerpc64"))]
pub(crate) const AT_HWCAP2: c_ulong = 26;
#[cfg(all(target_arch = "aarch64", target_os = "android"))]
pub(crate) const PROP_VALUE_MAX: c_int = 92;
}
pub(super) fn getauxval(type_: ffi::c_ulong) -> ffi::c_ulong {
#[cfg(all(target_arch = "aarch64", target_os = "android"))]
{
let mut arch = [0_u8; ffi::PROP_VALUE_MAX as usize];
let len = unsafe {
ffi::__system_property_get(
b"ro.arch\0".as_ptr().cast::<ffi::c_char>(),
arch.as_mut_ptr().cast::<ffi::c_char>(),
)
};
if len > 0 && arch.starts_with(b"exynos9810") {
return 0;
}
}
unsafe { ffi::getauxval(type_) }
}
}
#[cfg(target_os = "freebsd")]
mod os {
#[cfg_attr(test, allow(dead_code))]
pub(super) mod ffi {
pub(crate) use super::super::c_types::{c_int, c_ulong, c_void};
extern "C" {
pub(crate) fn elf_aux_info(aux: c_int, buf: *mut c_void, buf_len: c_int) -> c_int;
}
#[cfg(any(test, target_arch = "aarch64"))]
pub(crate) const AT_HWCAP: c_int = 25;
#[cfg(any(test, target_arch = "powerpc64"))]
pub(crate) const AT_HWCAP2: c_int = 26;
}
pub(super) fn getauxval(aux: ffi::c_int) -> ffi::c_ulong {
#[allow(clippy::cast_possible_wrap, clippy::cast_possible_truncation)]
const OUT_LEN: ffi::c_int = core::mem::size_of::<ffi::c_ulong>() as ffi::c_int;
let mut out: ffi::c_ulong = 0;
unsafe {
let res = ffi::elf_aux_info(
aux,
(&mut out as *mut ffi::c_ulong).cast::<ffi::c_void>(),
OUT_LEN,
);
debug_assert!(res == 0 || out == 0);
}
out
}
}
use arch::_detect;
#[cfg(target_arch = "aarch64")]
mod arch {
use super::{ffi, os, CpuInfo};
pub(super) const HWCAP_ATOMICS: ffi::c_ulong = 1 << 8;
pub(super) const HWCAP_USCAT: ffi::c_ulong = 1 << 25;
#[cfg(any(target_os = "linux", target_os = "android"))]
#[cfg(target_pointer_width = "64")]
#[cfg(test)]
pub(super) const HWCAP2_LRCPC3: ffi::c_ulong = 1 << 46;
#[cfg(any(target_os = "linux", target_os = "android"))]
#[cfg(target_pointer_width = "64")]
#[cfg(test)]
pub(super) const HWCAP2_LSE128: ffi::c_ulong = 1 << 47;
#[cold]
pub(super) fn _detect(info: &mut CpuInfo) {
let hwcap = os::getauxval(ffi::AT_HWCAP);
if hwcap & HWCAP_ATOMICS != 0 {
info.set(CpuInfo::HAS_LSE);
}
if hwcap & HWCAP_USCAT != 0 {
info.set(CpuInfo::HAS_LSE2);
}
#[cfg(any(target_os = "linux", target_os = "android"))]
#[cfg(target_pointer_width = "64")]
#[cfg(test)]
{
let hwcap2 = os::getauxval(ffi::AT_HWCAP2);
if hwcap2 & HWCAP2_LRCPC3 != 0 {
info.set(CpuInfo::HAS_RCPC3);
}
if hwcap2 & HWCAP2_LSE128 != 0 {
info.set(CpuInfo::HAS_LSE128);
}
}
}
}
#[cfg(target_arch = "powerpc64")]
mod arch {
use super::{ffi, os, CpuInfo};
pub(super) const PPC_FEATURE2_ARCH_2_07: ffi::c_ulong = 0x80000000;
#[cold]
pub(super) fn _detect(info: &mut CpuInfo) {
let hwcap2 = os::getauxval(ffi::AT_HWCAP2);
if hwcap2 & PPC_FEATURE2_ARCH_2_07 != 0 {
info.set(CpuInfo::HAS_QUADWORD_ATOMICS);
}
}
}
#[allow(
clippy::alloc_instead_of_core,
clippy::std_instead_of_alloc,
clippy::std_instead_of_core,
clippy::undocumented_unsafe_blocks,
clippy::wildcard_imports
)]
#[cfg(test)]
mod tests {
use super::*;
#[cfg(any(target_os = "linux", target_os = "android"))]
#[cfg(target_pointer_width = "64")]
#[test]
fn test_linux_like() {
use c_types::*;
use core::{arch::asm, mem};
use std::vec;
use test_helper::{libc, sys};
fn getauxval_pr_get_auxv(type_: ffi::c_ulong) -> Result<ffi::c_ulong, c_int> {
#[cfg(target_arch = "aarch64")]
unsafe fn prctl_get_auxv(out: *mut c_void, len: usize) -> Result<usize, c_int> {
let r: i64;
unsafe {
asm!(
"svc 0",
in("x8") sys::__NR_prctl as u64,
inout("x0") sys::PR_GET_AUXV as u64 => r,
in("x1") ptr_reg!(out),
in("x2") len as u64,
in("x3") 0_u64,
in("x4") 0_u64,
options(nostack, preserves_flags)
);
}
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
if (r as c_int) < 0 {
Err(r as c_int)
} else {
Ok(r as usize)
}
}
#[cfg(target_arch = "powerpc64")]
unsafe fn prctl_get_auxv(out: *mut c_void, len: usize) -> Result<usize, c_int> {
let r: i64;
unsafe {
asm!(
"sc",
"bns+ 2f",
"neg %r3, %r3",
"2:",
inout("r0") sys::__NR_prctl as u64 => _,
inout("r3") sys::PR_GET_AUXV as u64 => r,
inout("r4") ptr_reg!(out) => _,
inout("r5") len as u64 => _,
inout("r6") 0_u64 => _,
inout("r7") 0_u64 => _,
out("r8") _,
out("r9") _,
out("r10") _,
out("r11") _,
out("r12") _,
out("cr0") _,
options(nostack, preserves_flags)
);
}
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
if (r as c_int) < 0 {
Err(r as c_int)
} else {
Ok(r as usize)
}
}
let mut auxv = vec![unsafe { mem::zeroed::<sys::Elf64_auxv_t>() }; 38];
let old_len = auxv.len() * mem::size_of::<sys::Elf64_auxv_t>();
let _len = unsafe { prctl_get_auxv(auxv.as_mut_ptr().cast::<c_void>(), old_len)? };
for aux in &auxv {
if aux.a_type == type_ {
return Ok(unsafe { aux.a_un.a_val });
}
}
Err(0)
}
unsafe {
let mut u = mem::zeroed();
assert_eq!(libc::uname(&mut u), 0);
let release = std::ffi::CStr::from_ptr(u.release.as_ptr());
let release = core::str::from_utf8(release.to_bytes()).unwrap();
let mut digits = release.split('.');
let major = digits.next().unwrap().parse::<u32>().unwrap();
let minor = digits.next().unwrap().parse::<u32>().unwrap();
if (major, minor) < (6, 4) {
std::eprintln!("kernel version: {major}.{minor} (no pr_get_auxv)");
assert_eq!(getauxval_pr_get_auxv(ffi::AT_HWCAP).unwrap_err(), -22);
assert_eq!(getauxval_pr_get_auxv(ffi::AT_HWCAP2).unwrap_err(), -22);
} else {
std::eprintln!("kernel version: {major}.{minor} (has pr_get_auxv)");
assert_eq!(
os::getauxval(ffi::AT_HWCAP),
getauxval_pr_get_auxv(ffi::AT_HWCAP).unwrap()
);
assert_eq!(
os::getauxval(ffi::AT_HWCAP2),
getauxval_pr_get_auxv(ffi::AT_HWCAP2).unwrap()
);
}
}
}
#[allow(clippy::cast_sign_loss)]
#[cfg(all(target_arch = "aarch64", target_os = "android"))]
#[test]
fn test_android() {
unsafe {
let mut arch = [1; ffi::PROP_VALUE_MAX as usize];
let len = ffi::__system_property_get(
b"ro.arch\0".as_ptr().cast::<ffi::c_char>(),
arch.as_mut_ptr().cast::<ffi::c_char>(),
);
assert!(len >= 0);
std::eprintln!("len={}", len);
std::eprintln!("arch={:?}", arch);
std::eprintln!(
"arch={:?}",
core::str::from_utf8(core::slice::from_raw_parts(arch.as_ptr(), len as usize))
.unwrap()
);
}
}
#[allow(clippy::cast_possible_wrap)]
#[cfg(target_os = "freebsd")]
#[test]
fn test_freebsd() {
use c_types::*;
use core::{arch::asm, mem, ptr};
use test_helper::sys;
fn getauxval_sysctl_libc(type_: ffi::c_int) -> ffi::c_ulong {
let mut auxv: [sys::Elf64_Auxinfo; sys::AT_COUNT as usize] = unsafe { mem::zeroed() };
let mut len = core::mem::size_of_val(&auxv) as c_size_t;
let pid = unsafe { sys::getpid() };
let mib = [
sys::CTL_KERN as c_int,
sys::KERN_PROC as c_int,
sys::KERN_PROC_AUXV as c_int,
pid,
];
#[allow(clippy::cast_possible_truncation)]
let res = unsafe {
sys::sysctl(
mib.as_ptr(),
mib.len() as c_uint,
auxv.as_mut_ptr().cast::<c_void>(),
&mut len,
ptr::null_mut(),
0,
)
};
if res != -1 {
for aux in &auxv {
if aux.a_type == type_ as c_long {
return unsafe { aux.a_un.a_val as c_ulong };
}
}
}
0
}
fn getauxval_sysctl_asm_syscall(type_: ffi::c_int) -> Result<ffi::c_ulong, c_int> {
#[allow(non_camel_case_types)]
type pid_t = c_int;
#[cfg(target_arch = "aarch64")]
#[inline]
fn getpid() -> pid_t {
#[allow(clippy::cast_possible_truncation)]
unsafe {
let n = sys::SYS_getpid;
let r: i64;
asm!(
"svc 0",
in("x8") n as u64,
out("x0") r,
options(nostack, readonly),
);
r as pid_t
}
}
#[cfg(target_arch = "aarch64")]
#[inline]
unsafe fn sysctl(
name: *const c_int,
name_len: c_uint,
old_p: *mut c_void,
old_len_p: *mut c_size_t,
new_p: *const c_void,
new_len: c_size_t,
) -> Result<c_int, c_int> {
#[allow(clippy::cast_possible_truncation)]
unsafe {
let mut n = sys::SYS___sysctl as u64;
let r: i64;
asm!(
"svc 0",
"b.cc 2f",
"mov x8, x0",
"mov x0, #-1",
"2:",
inout("x8") n,
inout("x0") ptr_reg!(name) => r,
inout("x1") name_len as u64 => _,
in("x2") ptr_reg!(old_p),
in("x3") ptr_reg!(old_len_p),
in("x4") ptr_reg!(new_p),
in("x5") new_len as u64,
options(nostack),
);
if r as c_int == -1 {
Err(n as c_int)
} else {
Ok(r as c_int)
}
}
}
#[cfg(target_arch = "powerpc64")]
#[inline]
fn getpid() -> pid_t {
#[allow(clippy::cast_possible_truncation)]
unsafe {
let n = sys::SYS_getpid;
let r: i64;
asm!(
"sc",
inout("r0") n as u64 => _,
out("r3") r,
out("r4") _,
out("r5") _,
out("r6") _,
out("r7") _,
out("r8") _,
out("r9") _,
out("r10") _,
out("r11") _,
out("r12") _,
out("cr0") _,
options(nostack, preserves_flags, readonly),
);
r as pid_t
}
}
#[cfg(target_arch = "powerpc64")]
#[inline]
unsafe fn sysctl(
name: *const c_int,
name_len: c_uint,
old_p: *mut c_void,
old_len_p: *mut c_size_t,
new_p: *const c_void,
new_len: c_size_t,
) -> Result<c_int, c_int> {
#[allow(clippy::cast_possible_truncation)]
unsafe {
let mut n = sys::SYS___sysctl as u64;
let r: i64;
asm!(
"sc",
"bns+ 2f",
"mr %r0, %r3",
"li %r3, -1",
"2:",
inout("r0") n,
inout("r3") ptr_reg!(name) => r,
inout("r4") name_len as u64 => _,
inout("r5") ptr_reg!(old_p) => _,
inout("r6") ptr_reg!(old_len_p) => _,
inout("r7") ptr_reg!(new_p) => _,
inout("r8") new_len as u64 => _,
out("r9") _,
out("r10") _,
out("r11") _,
out("r12") _,
out("cr0") _,
options(nostack, preserves_flags)
);
if r as c_int == -1 {
Err(n as c_int)
} else {
Ok(r as c_int)
}
}
}
let mut auxv: [sys::Elf64_Auxinfo; sys::AT_COUNT as usize] = unsafe { mem::zeroed() };
let mut len = core::mem::size_of_val(&auxv) as c_size_t;
let pid = getpid();
let mib = [
sys::CTL_KERN as c_int,
sys::KERN_PROC as c_int,
sys::KERN_PROC_AUXV as c_int,
pid,
];
#[allow(clippy::cast_possible_truncation)]
unsafe {
sysctl(
mib.as_ptr(),
mib.len() as c_uint,
auxv.as_mut_ptr().cast::<c_void>(),
&mut len,
ptr::null_mut(),
0,
)?;
}
for aux in &auxv {
if aux.a_type == type_ as c_long {
return Ok(unsafe { aux.a_un.a_val as c_ulong });
}
}
Err(0)
}
assert_eq!(os::getauxval(ffi::AT_HWCAP), getauxval_sysctl_libc(ffi::AT_HWCAP));
assert_eq!(os::getauxval(ffi::AT_HWCAP2), getauxval_sysctl_libc(ffi::AT_HWCAP2));
assert_eq!(
os::getauxval(ffi::AT_HWCAP),
getauxval_sysctl_asm_syscall(ffi::AT_HWCAP).unwrap()
);
assert_eq!(
os::getauxval(ffi::AT_HWCAP2),
getauxval_sysctl_asm_syscall(ffi::AT_HWCAP2).unwrap_or(0)
);
}
#[allow(
clippy::cast_possible_wrap,
clippy::cast_sign_loss,
clippy::cast_possible_truncation,
clippy::no_effect_underscore_binding
)]
const _: fn() = || {
use test_helper::{libc, sys};
#[cfg(not(target_os = "freebsd"))]
type AtType = ffi::c_ulong;
#[cfg(target_os = "freebsd")]
type AtType = ffi::c_int;
#[cfg(any(target_os = "linux", target_os = "android"))]
{
let mut _getauxval: unsafe extern "C" fn(ffi::c_ulong) -> ffi::c_ulong = ffi::getauxval;
_getauxval = libc::getauxval;
_getauxval = sys::getauxval;
}
#[cfg(all(target_arch = "aarch64", target_os = "android"))]
{
let mut ___system_property_get: unsafe extern "C" fn(
*const ffi::c_char,
*mut ffi::c_char,
) -> ffi::c_int = ffi::__system_property_get;
___system_property_get = libc::__system_property_get;
___system_property_get = sys::__system_property_get;
static_assert!(ffi::PROP_VALUE_MAX == libc::PROP_VALUE_MAX);
static_assert!(ffi::PROP_VALUE_MAX == sys::PROP_VALUE_MAX as ffi::c_int);
}
#[cfg(target_os = "freebsd")]
{
let mut _elf_aux_info: unsafe extern "C" fn(
ffi::c_int,
*mut ffi::c_void,
ffi::c_int,
) -> ffi::c_int = ffi::elf_aux_info;
_elf_aux_info = libc::elf_aux_info;
_elf_aux_info = sys::elf_aux_info;
}
#[cfg(not(target_os = "freebsd"))] static_assert!(ffi::AT_HWCAP == libc::AT_HWCAP);
static_assert!(ffi::AT_HWCAP == sys::AT_HWCAP as AtType);
#[cfg(not(target_os = "freebsd"))] static_assert!(ffi::AT_HWCAP2 == libc::AT_HWCAP2);
static_assert!(ffi::AT_HWCAP2 == sys::AT_HWCAP2 as AtType);
#[cfg(target_arch = "aarch64")]
{
static_assert!(arch::HWCAP_ATOMICS == sys::HWCAP_ATOMICS as ffi::c_ulong);
static_assert!(arch::HWCAP_USCAT == sys::HWCAP_USCAT as ffi::c_ulong);
#[cfg(any(target_os = "linux", target_os = "android"))]
#[cfg(target_pointer_width = "64")]
{
static_assert!(arch::HWCAP2_LRCPC3 == sys::HWCAP2_LRCPC3 as ffi::c_ulong);
static_assert!(arch::HWCAP2_LSE128 == sys::HWCAP2_LSE128 as ffi::c_ulong);
}
}
#[cfg(target_arch = "powerpc64")]
{
static_assert!(
arch::PPC_FEATURE2_ARCH_2_07 == sys::PPC_FEATURE2_ARCH_2_07 as ffi::c_ulong
);
}
};
}