include!("common.rs");
use self::os::ffi;
#[cfg(any(target_os = "linux", target_os = "android"))]
mod os {
#[cfg_attr(test, allow(dead_code))]
pub(super) mod ffi {
pub(crate) use crate::utils::ffi::c_ulong;
#[allow(unused_imports)]
pub(crate) use crate::utils::ffi::{c_char, c_int, c_void};
sys_const!({
pub(crate) const AT_HWCAP: c_ulong = 16;
#[cfg(any(
test,
all(target_arch = "aarch64", target_pointer_width = "64"),
target_arch = "powerpc64",
))]
pub(crate) const AT_HWCAP2: c_ulong = 26;
#[cfg(any(
test,
not(any(
all(
target_os = "linux",
any(
all(
target_env = "gnu",
any(
target_arch = "aarch64",
all(target_arch = "powerpc64", target_endian = "little"),
),
),
target_env = "musl",
target_env = "ohos",
),
),
all(target_os = "android", target_pointer_width = "64"),
portable_atomic_outline_atomics,
)),
))]
pub(crate) const RTLD_DEFAULT: *mut c_void = core::ptr::null_mut();
#[cfg(all(target_arch = "aarch64", target_os = "android"))]
pub(crate) const PROP_VALUE_MAX: c_int = 92;
});
sys_fn!({
extern "C" {
#[cfg(any(
test,
all(
target_os = "linux",
any(
all(
target_env = "gnu",
any(
target_arch = "aarch64",
all(target_arch = "powerpc64", target_endian = "little"),
),
),
target_env = "musl",
target_env = "ohos",
),
),
all(target_os = "android", target_pointer_width = "64"),
portable_atomic_outline_atomics,
))]
pub(crate) fn getauxval(type_: c_ulong) -> c_ulong;
#[cfg(any(
test,
not(any(
all(
target_os = "linux",
any(
all(
target_env = "gnu",
any(
target_arch = "aarch64",
all(target_arch = "powerpc64", target_endian = "little"),
),
),
target_env = "musl",
target_env = "ohos",
),
),
all(target_os = "android", target_pointer_width = "64"),
portable_atomic_outline_atomics,
)),
))]
pub(crate) fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void;
#[cfg(all(target_arch = "aarch64", target_os = "android"))]
pub(crate) fn __system_property_get(
name: *const c_char,
value: *mut c_char,
) -> c_int;
}
});
}
pub(super) type GetauxvalTy = unsafe extern "C" fn(ffi::c_ulong) -> ffi::c_ulong;
pub(super) fn getauxval(type_: ffi::c_ulong) -> ffi::c_ulong {
#[cfg(any(
all(
target_os = "linux",
any(
all(
target_env = "gnu",
any(
target_arch = "aarch64",
all(target_arch = "powerpc64", target_endian = "little"),
),
),
target_env = "musl",
target_env = "ohos",
),
),
all(target_os = "android", target_pointer_width = "64"),
portable_atomic_outline_atomics,
))]
let getauxval: GetauxvalTy = ffi::getauxval;
#[cfg(not(any(
all(
target_os = "linux",
any(
all(
target_env = "gnu",
any(
target_arch = "aarch64",
all(target_arch = "powerpc64", target_endian = "little"),
),
),
target_env = "musl",
target_env = "ohos",
),
),
all(target_os = "android", target_pointer_width = "64"),
portable_atomic_outline_atomics,
)))]
let getauxval: GetauxvalTy = unsafe {
let ptr = ffi::dlsym(ffi::RTLD_DEFAULT, c!("getauxval").as_ptr());
if ptr.is_null() {
return 0;
}
core::mem::transmute::<*mut ffi::c_void, GetauxvalTy>(ptr)
};
unsafe { getauxval(type_) }
}
}
#[cfg(any(target_os = "freebsd", target_os = "openbsd"))]
mod os {
use core::mem;
#[cfg_attr(test, allow(dead_code))]
pub(super) mod ffi {
#[allow(unused_imports)]
pub(crate) use crate::utils::ffi::c_char;
pub(crate) use crate::utils::ffi::{c_int, c_ulong, c_void};
sys_const!({
pub(crate) const AT_HWCAP: c_int = 25;
#[cfg(any(
test,
all(target_os = "freebsd", target_arch = "aarch64", target_pointer_width = "64"),
target_arch = "powerpc64",
))]
pub(crate) const AT_HWCAP2: c_int = 26;
#[cfg(any(
test,
not(any(
all(target_os = "freebsd", target_arch = "aarch64"),
portable_atomic_outline_atomics,
)),
))]
#[allow(clippy::cast_sign_loss)]
pub(crate) const RTLD_DEFAULT: *mut c_void = -2_isize as usize as *mut c_void;
});
sys_fn!({
extern "C" {
#[cfg(any(
test,
any(
all(target_os = "freebsd", target_arch = "aarch64"),
portable_atomic_outline_atomics,
),
))]
pub(crate) fn elf_aux_info(aux: c_int, buf: *mut c_void, buf_len: c_int) -> c_int;
#[cfg(any(
test,
not(any(
all(target_os = "freebsd", target_arch = "aarch64"),
portable_atomic_outline_atomics,
)),
))]
pub(crate) fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void;
}
});
}
pub(super) type ElfAuxInfoTy =
unsafe extern "C" fn(ffi::c_int, *mut ffi::c_void, ffi::c_int) -> ffi::c_int;
pub(super) fn getauxval(aux: ffi::c_int) -> ffi::c_ulong {
#[allow(clippy::cast_possible_wrap, clippy::cast_possible_truncation)]
const OUT_LEN: ffi::c_int = mem::size_of::<ffi::c_ulong>() as ffi::c_int;
#[cfg(any(
all(target_os = "freebsd", target_arch = "aarch64"),
portable_atomic_outline_atomics,
))]
let elf_aux_info: ElfAuxInfoTy = ffi::elf_aux_info;
#[cfg(not(any(
all(target_os = "freebsd", target_arch = "aarch64"),
portable_atomic_outline_atomics,
)))]
let elf_aux_info: ElfAuxInfoTy = unsafe {
let ptr = ffi::dlsym(ffi::RTLD_DEFAULT, c!("elf_aux_info").as_ptr());
if ptr.is_null() {
return 0;
}
mem::transmute::<*mut ffi::c_void, ElfAuxInfoTy>(ptr)
};
let mut out: ffi::c_ulong = 0;
let res = unsafe {
elf_aux_info(aux, (&mut out as *mut ffi::c_ulong).cast::<ffi::c_void>(), OUT_LEN)
};
debug_assert!(res == 0 || out == 0);
out
}
}
use self::arch::_detect;
#[cfg(target_arch = "aarch64")]
mod arch {
use super::{CpuInfo, ffi, os};
sys_const!({
pub(super) const HWCAP_ATOMICS: ffi::c_ulong = 1 << 8;
pub(super) const HWCAP_USCAT: ffi::c_ulong = 1 << 25;
#[cfg(not(target_os = "openbsd"))]
#[cfg(target_pointer_width = "64")]
pub(super) const HWCAP2_LRCPC3: ffi::c_ulong = 1 << 46;
#[cfg(not(target_os = "openbsd"))]
#[cfg(target_pointer_width = "64")]
pub(super) const HWCAP2_LSE128: ffi::c_ulong = 1 << 47;
});
#[cold]
pub(super) fn _detect(info: &mut CpuInfo) {
#[cfg(target_os = "android")]
{
let mut arch = [0_u8; ffi::PROP_VALUE_MAX as usize];
let len = unsafe {
ffi::__system_property_get(
c!("ro.arch").as_ptr(),
arch.as_mut_ptr().cast::<ffi::c_char>(),
)
};
if len > 0 && arch.starts_with(b"exynos9810") {
return;
}
}
let hwcap = os::getauxval(ffi::AT_HWCAP);
if hwcap & HWCAP_ATOMICS != 0 {
info.set(CpuInfo::HAS_LSE);
}
if hwcap & HWCAP_USCAT != 0 {
info.set(CpuInfo::HAS_LSE2);
}
#[cfg(not(target_os = "openbsd"))]
#[cfg(target_pointer_width = "64")]
{
let hwcap2 = os::getauxval(ffi::AT_HWCAP2);
if hwcap2 & HWCAP2_LRCPC3 != 0 {
info.set(CpuInfo::HAS_RCPC3);
}
if hwcap2 & HWCAP2_LSE128 != 0 {
info.set(CpuInfo::HAS_LSE128);
}
}
}
}
#[cfg(target_arch = "powerpc64")]
mod arch {
use super::{CpuInfo, ffi, os};
sys_const!({
pub(super) const PPC_FEATURE_BOOKE: ffi::c_ulong = 0x00008000;
pub(super) const PPC_FEATURE2_ARCH_2_07: ffi::c_ulong = 0x80000000;
pub(super) const PPC_FEATURE2_ARCH_3_00: ffi::c_ulong = 0x00800000;
pub(super) const PPC_FEATURE2_ARCH_3_1: ffi::c_ulong = 0x00040000;
});
#[cold]
pub(super) fn _detect(info: &mut CpuInfo) {
let hwcap = os::getauxval(ffi::AT_HWCAP);
if hwcap & PPC_FEATURE_BOOKE != 0 {
return;
}
let hwcap2 = os::getauxval(ffi::AT_HWCAP2);
let isa_2_07_or_later =
PPC_FEATURE2_ARCH_2_07 | PPC_FEATURE2_ARCH_3_00 | PPC_FEATURE2_ARCH_3_1;
if hwcap2 & isa_2_07_or_later != 0 {
info.set(CpuInfo::HAS_QUADWORD_ATOMICS);
}
}
}
#[allow(
clippy::alloc_instead_of_core,
clippy::std_instead_of_alloc,
clippy::std_instead_of_core,
clippy::undocumented_unsafe_blocks,
clippy::wildcard_imports
)]
#[cfg(test)]
mod tests {
use std::mem;
use super::*;
#[allow(clippy::cast_sign_loss)]
#[cfg(all(target_arch = "aarch64", target_os = "android"))]
#[test]
fn test_android() {
use std::{eprintln, slice, str};
unsafe {
let mut arch = [1; ffi::PROP_VALUE_MAX as usize];
let len = ffi::__system_property_get(
c!("ro.arch").as_ptr(),
arch.as_mut_ptr().cast::<ffi::c_char>(),
);
assert!(len >= 0);
eprintln!("len={}", len);
eprintln!("arch={:?}", arch);
eprintln!(
"arch={:?}",
str::from_utf8(slice::from_raw_parts(arch.as_ptr(), len as usize)).unwrap()
);
}
}
#[cfg(any(target_os = "linux", target_os = "android"))]
#[test]
fn test_dlsym_getauxval() {
unsafe {
let ptr = ffi::dlsym(ffi::RTLD_DEFAULT, c!("getauxval").as_ptr());
if cfg!(any(
all(
target_os = "linux",
any(
target_env = "gnu",
all(
any(target_env = "musl", target_env = "ohos"),
not(target_feature = "crt-static"),
),
),
),
target_os = "android",
)) {
assert!(!ptr.is_null());
} else if option_env!("CI").is_some() {
assert!(ptr.is_null());
}
if ptr.is_null() {
return;
}
let dlsym_getauxval = mem::transmute::<*mut ffi::c_void, os::GetauxvalTy>(ptr);
assert_eq!(dlsym_getauxval(ffi::AT_HWCAP), ffi::getauxval(ffi::AT_HWCAP));
assert_eq!(dlsym_getauxval(ffi::AT_HWCAP2), ffi::getauxval(ffi::AT_HWCAP2));
}
}
#[cfg(any(target_os = "freebsd", target_os = "openbsd"))]
#[test]
fn test_dlsym_elf_aux_info() {
unsafe {
let ptr = ffi::dlsym(ffi::RTLD_DEFAULT, c!("elf_aux_info").as_ptr());
if cfg!(target_os = "freebsd") || option_env!("CI").is_some() {
assert!(!ptr.is_null());
}
if ptr.is_null() {
return;
}
let dlsym_elf_aux_info = mem::transmute::<*mut ffi::c_void, os::ElfAuxInfoTy>(ptr);
let mut out: ffi::c_ulong = 0;
let mut dlsym_out: ffi::c_ulong = 0;
#[allow(clippy::cast_possible_wrap, clippy::cast_possible_truncation)]
let out_len = mem::size_of::<ffi::c_ulong>() as ffi::c_int;
assert_eq!(
ffi::elf_aux_info(
ffi::AT_HWCAP,
(&mut out as *mut ffi::c_ulong).cast::<ffi::c_void>(),
out_len,
),
dlsym_elf_aux_info(
ffi::AT_HWCAP,
(&mut dlsym_out as *mut ffi::c_ulong).cast::<ffi::c_void>(),
out_len,
),
);
assert_eq!(out, dlsym_out);
out = 0;
dlsym_out = 0;
assert_eq!(
ffi::elf_aux_info(
ffi::AT_HWCAP2,
(&mut out as *mut ffi::c_ulong).cast::<ffi::c_void>(),
out_len,
),
dlsym_elf_aux_info(
ffi::AT_HWCAP2,
(&mut dlsym_out as *mut ffi::c_ulong).cast::<ffi::c_void>(),
out_len,
),
);
assert_eq!(out, dlsym_out);
}
}
#[cfg(any(target_os = "linux", target_os = "android"))]
#[cfg(not(all(target_arch = "aarch64", target_pointer_width = "32")))]
#[test]
fn test_alternative() {
use crate::utils::ffi::*;
#[cfg(not(portable_atomic_no_asm))]
use std::arch::asm;
use std::{str, vec};
#[cfg(target_pointer_width = "32")]
use sys::Elf32_auxv_t as Elf_auxv_t;
#[cfg(target_pointer_width = "64")]
use sys::Elf64_auxv_t as Elf_auxv_t;
use test_helper::sys;
fn getauxval_pr_get_auxv_no_libc(type_: c_ulong) -> Result<c_ulong, c_int> {
#[cfg(target_arch = "aarch64")]
unsafe fn prctl_get_auxv(out: *mut c_void, len: usize) -> Result<usize, c_int> {
let r: i64;
unsafe {
asm!(
"svc 0",
in("x8") sys::__NR_prctl as u64,
inout("x0") sys::PR_GET_AUXV as u64 => r,
in("x1") ptr_reg!(out),
in("x2") len as u64,
in("x3") 0_u64,
in("x4") 0_u64,
options(nostack, preserves_flags),
);
}
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
if (r as c_int) < 0 { Err(r as c_int) } else { Ok(r as usize) }
}
#[cfg(target_arch = "powerpc64")]
unsafe fn prctl_get_auxv(out: *mut c_void, len: usize) -> Result<usize, c_int> {
let r: i64;
unsafe {
asm!(
"sc",
"bns+ 2f",
"neg %r3, %r3",
"2:",
inout("r0") sys::__NR_prctl as u64 => _,
inout("r3") sys::PR_GET_AUXV as u64 => r,
inout("r4") ptr_reg!(out) => _,
inout("r5") len as u64 => _,
inout("r6") 0_u64 => _,
inout("r7") 0_u64 => _,
out("r8") _,
out("r9") _,
out("r10") _,
out("r11") _,
out("r12") _,
out("cr0") _,
options(nostack, preserves_flags),
);
}
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
if (r as c_int) < 0 { Err(r as c_int) } else { Ok(r as usize) }
}
let mut auxv = vec![unsafe { mem::zeroed::<Elf_auxv_t>() }; 38];
let old_len = auxv.len() * mem::size_of::<Elf_auxv_t>();
let _len = unsafe { prctl_get_auxv(auxv.as_mut_ptr().cast::<c_void>(), old_len)? };
for aux in &auxv {
if aux.a_type == type_ {
return Ok(unsafe { aux.a_un.a_val });
}
}
Err(0)
}
fn getauxval_pr_get_auxv_libc(type_: c_ulong) -> Result<c_ulong, c_int> {
unsafe fn prctl_get_auxv(out: *mut c_void, len: usize) -> Result<usize, c_int> {
#[allow(clippy::cast_possible_wrap)]
let r = unsafe { libc::prctl(sys::PR_GET_AUXV as c_int, out, len, 0, 0) };
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
if (r as c_int) < 0 { Err(r as c_int) } else { Ok(r as usize) }
}
let mut auxv = vec![unsafe { mem::zeroed::<Elf_auxv_t>() }; 38];
let old_len = auxv.len() * mem::size_of::<Elf_auxv_t>();
let _len = unsafe { prctl_get_auxv(auxv.as_mut_ptr().cast::<c_void>(), old_len)? };
for aux in &auxv {
if aux.a_type == type_ {
return Ok(unsafe { aux.a_un.a_val });
}
}
Err(0)
}
unsafe {
let mut u = mem::zeroed();
assert_eq!(libc::uname(&mut u), 0);
let release = std::ffi::CStr::from_ptr(u.release.as_ptr());
let release = str::from_utf8(release.to_bytes()).unwrap();
let mut digits = release.split('.');
let major = digits.next().unwrap().parse::<u32>().unwrap();
let minor = digits.next().unwrap().parse::<u32>().unwrap();
if (major, minor) < (6, 4) || cfg!(qemu) {
std::eprintln!("kernel version: {}.{} (no pr_get_auxv)", major, minor);
assert_eq!(getauxval_pr_get_auxv_libc(ffi::AT_HWCAP).unwrap_err(), -1);
assert_eq!(getauxval_pr_get_auxv_libc(ffi::AT_HWCAP2).unwrap_err(), -1);
assert_eq!(
getauxval_pr_get_auxv_no_libc(ffi::AT_HWCAP).unwrap_err(),
-libc::EINVAL
);
assert_eq!(
getauxval_pr_get_auxv_no_libc(ffi::AT_HWCAP2).unwrap_err(),
-libc::EINVAL
);
} else if cfg!(valgrind) {
} else {
std::eprintln!("kernel version: {}.{} (has pr_get_auxv)", major, minor);
assert_eq!(
os::getauxval(ffi::AT_HWCAP),
getauxval_pr_get_auxv_libc(ffi::AT_HWCAP).unwrap()
);
assert_eq!(
os::getauxval(ffi::AT_HWCAP2),
getauxval_pr_get_auxv_libc(ffi::AT_HWCAP2).unwrap()
);
assert_eq!(
os::getauxval(ffi::AT_HWCAP),
getauxval_pr_get_auxv_no_libc(ffi::AT_HWCAP).unwrap()
);
assert_eq!(
os::getauxval(ffi::AT_HWCAP2),
getauxval_pr_get_auxv_no_libc(ffi::AT_HWCAP2).unwrap()
);
}
}
}
#[allow(clippy::cast_possible_wrap)]
#[cfg(target_os = "freebsd")]
#[test]
fn test_alternative() {
use crate::utils::ffi::*;
#[cfg(not(portable_atomic_no_asm))]
use std::arch::asm;
use std::ptr;
use test_helper::sys;
fn getauxval_sysctl_libc(type_: ffi::c_int) -> Result<ffi::c_ulong, c_int> {
let mut auxv: [sys::Elf_Auxinfo; sys::AT_COUNT as usize] = unsafe { mem::zeroed() };
let mut len = mem::size_of_val(&auxv) as c_size_t;
let pid = unsafe { libc::getpid() };
let mib = [
sys::CTL_KERN as c_int,
sys::KERN_PROC as c_int,
sys::KERN_PROC_AUXV as c_int,
pid,
];
#[allow(clippy::cast_possible_truncation)]
let res = unsafe {
libc::sysctl(
mib.as_ptr(),
mib.len() as c_uint,
auxv.as_mut_ptr().cast::<c_void>(),
&mut len,
ptr::null_mut(),
0,
)
};
if res == -1 {
return Err(res);
}
for aux in &auxv {
if aux.a_type == type_ as c_long {
return Ok(unsafe { aux.a_un.a_val as c_ulong });
}
}
Err(0)
}
fn getauxval_sysctl_no_libc(type_: ffi::c_int) -> Result<ffi::c_ulong, c_int> {
#[allow(non_camel_case_types)]
type pid_t = c_int;
#[cfg(target_arch = "aarch64")]
#[inline]
fn getpid() -> pid_t {
#[allow(clippy::cast_possible_truncation)]
unsafe {
let n = sys::SYS_getpid;
let r: i64;
asm!(
"svc 0",
in("x8") n as u64,
out("x0") r,
options(nostack, readonly),
);
r as pid_t
}
}
#[cfg(target_arch = "aarch64")]
#[inline]
unsafe fn sysctl(
name: *const c_int,
name_len: c_uint,
old_p: *mut c_void,
old_len_p: *mut c_size_t,
new_p: *const c_void,
new_len: c_size_t,
) -> Result<c_int, c_int> {
#[allow(clippy::cast_possible_truncation)]
unsafe {
let mut n = sys::SYS___sysctl as u64;
let r: i64;
asm!(
"svc 0",
"b.cc 2f",
"mov x8, x0",
"mov x0, #-1",
"2:",
inout("x8") n,
inout("x0") ptr_reg!(name) => r,
inout("x1") name_len as u64 => _,
in("x2") ptr_reg!(old_p),
in("x3") ptr_reg!(old_len_p),
in("x4") ptr_reg!(new_p),
in("x5") new_len as u64,
options(nostack),
);
if r as c_int == -1 { Err(n as c_int) } else { Ok(r as c_int) }
}
}
#[cfg(target_arch = "powerpc64")]
#[inline]
fn getpid() -> pid_t {
#[allow(clippy::cast_possible_truncation)]
unsafe {
let n = sys::SYS_getpid;
let r: i64;
asm!(
"sc",
inout("r0") n as u64 => _,
out("r3") r,
out("r4") _,
out("r5") _,
out("r6") _,
out("r7") _,
out("r8") _,
out("r9") _,
out("r10") _,
out("r11") _,
out("r12") _,
out("cr0") _,
options(nostack, preserves_flags, readonly),
);
r as pid_t
}
}
#[cfg(target_arch = "powerpc64")]
#[inline]
unsafe fn sysctl(
name: *const c_int,
name_len: c_uint,
old_p: *mut c_void,
old_len_p: *mut c_size_t,
new_p: *const c_void,
new_len: c_size_t,
) -> Result<c_int, c_int> {
#[allow(clippy::cast_possible_truncation)]
unsafe {
let mut n = sys::SYS___sysctl as u64;
let r: i64;
asm!(
"sc",
"bns+ 2f",
"mr %r0, %r3",
"li %r3, -1",
"2:",
inout("r0") n,
inout("r3") ptr_reg!(name) => r,
inout("r4") name_len as u64 => _,
inout("r5") ptr_reg!(old_p) => _,
inout("r6") ptr_reg!(old_len_p) => _,
inout("r7") ptr_reg!(new_p) => _,
inout("r8") new_len as u64 => _,
out("r9") _,
out("r10") _,
out("r11") _,
out("r12") _,
out("cr0") _,
options(nostack, preserves_flags),
);
if r as c_int == -1 { Err(n as c_int) } else { Ok(r as c_int) }
}
}
let mut auxv: [sys::Elf_Auxinfo; sys::AT_COUNT as usize] = unsafe { mem::zeroed() };
let mut len = mem::size_of_val(&auxv) as c_size_t;
let pid = getpid();
let mib = [
sys::CTL_KERN as c_int,
sys::KERN_PROC as c_int,
sys::KERN_PROC_AUXV as c_int,
pid,
];
#[allow(clippy::cast_possible_truncation)]
unsafe {
sysctl(
mib.as_ptr(),
mib.len() as c_uint,
auxv.as_mut_ptr().cast::<c_void>(),
&mut len,
ptr::null_mut(),
0,
)?;
}
for aux in &auxv {
if aux.a_type == type_ as c_long {
return Ok(unsafe { aux.a_un.a_val as c_ulong });
}
}
Err(0)
}
let hwcap2_else = |e| if cfg!(target_arch = "aarch64") { 0 } else { panic!("{:?}", e) };
assert_eq!(os::getauxval(ffi::AT_HWCAP), getauxval_sysctl_libc(ffi::AT_HWCAP).unwrap());
assert_eq!(
os::getauxval(ffi::AT_HWCAP2),
getauxval_sysctl_libc(ffi::AT_HWCAP2).unwrap_or_else(hwcap2_else)
);
assert_eq!(os::getauxval(ffi::AT_HWCAP), getauxval_sysctl_no_libc(ffi::AT_HWCAP).unwrap());
assert_eq!(
os::getauxval(ffi::AT_HWCAP2),
getauxval_sysctl_no_libc(ffi::AT_HWCAP2).unwrap_or_else(hwcap2_else)
);
}
}