use core::sync::atomic::{AtomicBool, Ordering};
#[cfg(target_os = "linux")]
pub(crate) use sys::count;
#[cfg(not(target_os = "linux"))]
#[allow(unused_imports)]
pub(crate) use sys::count;
pub(crate) use sys::{nth, read_here, set_here, set_here_to, CpuMask};
pub(crate) fn available() -> usize {
#[cfg(target_os = "linux")]
if let Some(set) = read_here() {
let n = count(&set);
if n > 0 {
return n;
}
}
std::thread::available_parallelism().map_or(1, |n| n.get())
}
pub(crate) fn oversubscribed(participants: usize) -> bool {
participants > available()
}
pub(crate) fn pin_to(set: &CpuMask, k: usize) {
if let Some(cpu) = nth(set, k) {
set_here_to(cpu);
}
}
#[derive(Clone, Copy)]
struct Placed {
wide: CpuMask,
home: usize,
}
thread_local! {
static PLACED: core::cell::Cell<Option<Placed>> = const { core::cell::Cell::new(None) };
}
pub(crate) struct LeaderPin {
wide: CpuMask,
prev: Option<Placed>,
}
impl LeaderPin {
pub(crate) fn enter(wide: &CpuMask, home: usize) -> Option<LeaderPin> {
if !set_here_to(home) {
return None;
}
let prev = PLACED.with(|p| p.replace(Some(Placed { wide: *wide, home })));
Some(LeaderPin { wide: *wide, prev })
}
}
impl Drop for LeaderPin {
fn drop(&mut self) {
let _ = set_here(&self.wide);
PLACED.with(|p| p.set(self.prev));
}
}
pub(crate) struct TeamPin {
pin: core::mem::ManuallyDrop<LeaderPin>,
thread: std::thread::ThreadId,
}
impl TeamPin {
pub(crate) fn enter(participants: usize) -> Option<TeamPin> {
let set = read_here()?;
if count(&set) < participants {
return None;
}
let pin = LeaderPin::enter(&set, nth(&set, 0)?)?;
Some(TeamPin {
pin: core::mem::ManuallyDrop::new(pin),
thread: std::thread::current().id(),
})
}
pub(crate) fn set_here(&self) -> Option<&CpuMask> {
(std::thread::current().id() == self.thread).then_some(&self.pin.wide)
}
}
impl Drop for TeamPin {
fn drop(&mut self) {
if std::thread::current().id() == self.thread {
unsafe { core::mem::ManuallyDrop::drop(&mut self.pin) };
}
}
}
pub(crate) fn widened<R>(f: impl FnOnce() -> R) -> R {
struct Restore(usize);
impl Drop for Restore {
fn drop(&mut self) {
let _ = set_here_to(self.0);
}
}
let placed = PLACED.with(|p| p.get());
let _restore = placed.and_then(|p| set_here(&p.wide).then_some(Restore(p.home)));
f()
}
#[cfg_attr(not(feature = "parallel"), allow(dead_code))]
pub(crate) struct SavedMask {
mask: core::cell::UnsafeCell<core::mem::MaybeUninit<CpuMask>>,
pinned: AtomicBool,
}
unsafe impl Sync for SavedMask {}
#[cfg_attr(not(feature = "parallel"), allow(dead_code))]
impl SavedMask {
pub(crate) const fn new() -> Self {
SavedMask {
mask: core::cell::UnsafeCell::new(core::mem::MaybeUninit::uninit()),
pinned: AtomicBool::new(false),
}
}
pub(crate) fn save_if_unpinned(&self) {
if self.pinned.load(Ordering::Relaxed) {
return;
}
if let Some(mask) = read_here() {
unsafe { (*self.mask.get()).write(mask) };
self.pinned.store(true, Ordering::Relaxed);
}
}
pub(crate) fn restore(&self) {
if !self.pinned.load(Ordering::Relaxed) {
return;
}
let mask = unsafe { (*self.mask.get()).assume_init_ref() };
if set_here(mask) {
self.pinned.store(false, Ordering::Relaxed);
}
}
}
#[cfg(target_os = "linux")]
mod sys {
use core::mem;
const SETSIZE: usize = 8 * mem::size_of::<libc::cpu_set_t>();
#[derive(Clone, Copy)]
pub(crate) struct CpuMask {
set: libc::cpu_set_t,
}
pub(crate) fn read_here() -> Option<CpuMask> {
let mut set = unsafe { mem::zeroed::<libc::cpu_set_t>() };
let r = unsafe { libc::sched_getaffinity(0, mem::size_of::<libc::cpu_set_t>(), &mut set) };
(r == 0).then_some(CpuMask { set })
}
pub(crate) fn set_here(mask: &CpuMask) -> bool {
let r = unsafe { libc::sched_setaffinity(0, mem::size_of::<libc::cpu_set_t>(), &mask.set) };
r == 0
}
pub(crate) fn set_here_to(cpu: usize) -> bool {
if cpu >= SETSIZE {
return false;
}
unsafe {
let mut set = mem::zeroed::<libc::cpu_set_t>();
libc::CPU_ZERO(&mut set);
libc::CPU_SET(cpu, &mut set);
set_here(&CpuMask { set })
}
}
pub(crate) fn count(mask: &CpuMask) -> usize {
let n = unsafe { libc::CPU_COUNT(&mask.set) };
n.max(0) as usize
}
pub(crate) fn nth(mask: &CpuMask, k: usize) -> Option<usize> {
if k >= count(mask) {
return None;
}
let mut want = k;
for cpu in 0..SETSIZE {
if unsafe { libc::CPU_ISSET(cpu, &mask.set) } {
if want == 0 {
return Some(cpu);
}
want -= 1;
}
}
None
}
}
#[cfg(not(target_os = "linux"))]
#[allow(dead_code)] mod sys {
#[derive(Clone, Copy)]
pub(crate) struct CpuMask(());
pub(crate) fn read_here() -> Option<CpuMask> {
None
}
pub(crate) fn set_here(_mask: &CpuMask) -> bool {
false
}
pub(crate) fn set_here_to(_cpu: usize) -> bool {
false
}
pub(crate) fn count(_mask: &CpuMask) -> usize {
0
}
pub(crate) fn nth(_mask: &CpuMask, _k: usize) -> Option<usize> {
None
}
}