use core::sync::atomic::{AtomicBool, Ordering};
#[cfg(target_os = "linux")]
pub(crate) use sys::count;
#[cfg(not(target_os = "linux"))]
#[allow(unused_imports)]
pub(crate) use sys::count;
pub(crate) use sys::{contains, cpus, read_here, subset, CpuMask};
pub(crate) fn available() -> usize {
#[cfg(target_os = "linux")]
if let Some(placed) = PLACED.with(core::cell::Cell::get) {
let n = count(&placed.wide);
if n > 0 {
return n;
}
}
AVAILABLE.with(|memo| {
let cached = memo.get();
if cached != 0 {
return cached;
}
let n = unplaced_count();
memo.set(n);
n
})
}
fn unplaced_count() -> usize {
#[cfg(target_os = "linux")]
if let Some(set) = read_here() {
let n = count(&set);
if n > 0 {
return n;
}
}
std::thread::available_parallelism().map_or(1, |n| n.get())
}
thread_local! {
static AVAILABLE: core::cell::Cell<usize> = const { core::cell::Cell::new(0) };
}
pub(crate) fn set_here(mask: &CpuMask) -> bool {
let moved = sys::set_here(mask);
if moved {
AVAILABLE.with(|memo| memo.set(0));
}
moved
}
pub(crate) fn note_count(n: usize) {
if n > 0 {
AVAILABLE.with(|memo| memo.set(n));
}
}
pub(crate) fn set_here_to(cpu: usize) -> bool {
let moved = sys::set_here_to(cpu);
if moved {
AVAILABLE.with(|memo| memo.set(0));
}
moved
}
pub(crate) fn oversubscribed(participants: usize) -> bool {
participants > available()
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
pub enum PinStrategy {
#[default]
Spread,
Pack,
}
pub(crate) const MAX_CPUS: usize = 1024;
const UNKNOWN: usize = MAX_CPUS;
#[derive(Debug, Default)]
pub(crate) struct Topology {
node_of: Box<[u16]>,
}
impl Topology {
#[cfg_attr(not(any(target_os = "linux", test)), allow(dead_code))]
pub(crate) fn from_nodes(nodes: &[Vec<usize>]) -> Self {
let width = nodes
.iter()
.flatten()
.copied()
.filter(|&cpu| cpu < MAX_CPUS)
.max()
.map_or(0, |cpu| cpu + 1);
let mut node_of = vec![u16::MAX; width].into_boxed_slice();
for (node, cpus) in nodes.iter().enumerate().take(MAX_CPUS) {
for &cpu in cpus {
if let Some(slot) = node_of.get_mut(cpu) {
*slot = node as u16;
}
}
}
Topology { node_of }
}
fn group(&self, cpu: u16) -> usize {
match self.node_of.get(usize::from(cpu)) {
Some(&node) if node != u16::MAX => usize::from(node),
_ => UNKNOWN,
}
}
}
pub(crate) fn host_topology() -> &'static Topology {
#[cfg(test)]
if let Some(topology) = TEST_TOPOLOGY.with(std::cell::Cell::get) {
return topology;
}
static HOST: std::sync::OnceLock<Topology> = std::sync::OnceLock::new();
HOST.get_or_init(read_host_topology)
}
#[cfg(test)]
std::thread_local! {
static TEST_TOPOLOGY: std::cell::Cell<Option<&'static Topology>> =
const { std::cell::Cell::new(None) };
}
#[cfg(test)]
#[cfg_attr(
not(all(feature = "parallel", target_os = "linux")),
allow(dead_code) // only the placement test (Linux, `parallel`) sets it
)]
pub(crate) fn set_test_topology(topology: Option<&'static Topology>) {
TEST_TOPOLOGY.with(|slot| slot.set(topology));
}
#[cfg(target_os = "linux")]
fn read_host_topology() -> Topology {
let Ok(dir) = std::fs::read_dir("/sys/devices/system/node") else {
return Topology::default();
};
let mut nodes: Vec<Vec<usize>> = Vec::new();
for entry in dir.flatten() {
let name = entry.file_name();
let Some(id) = name
.to_str()
.and_then(|n| n.strip_prefix("node"))
.and_then(|n| n.parse::<usize>().ok())
else {
continue;
};
if id >= MAX_CPUS {
continue;
}
let Ok(list) = std::fs::read_to_string(entry.path().join("cpulist")) else {
continue;
};
if nodes.len() <= id {
nodes.resize_with(id + 1, Vec::new);
}
nodes[id] = parse_cpulist(&list);
}
Topology::from_nodes(&nodes)
}
#[cfg(not(target_os = "linux"))]
fn read_host_topology() -> Topology {
Topology::default()
}
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
fn parse_cpulist(list: &str) -> Vec<usize> {
let mut cpus = Vec::new();
for piece in list.trim().split(',').filter(|p| !p.is_empty()) {
let mut ends = piece.splitn(2, '-').map(|n| n.trim().parse::<usize>());
match (ends.next(), ends.next()) {
(Some(Ok(lo)), None) => cpus.push(lo),
(Some(Ok(lo)), Some(Ok(hi))) if lo <= hi => {
cpus.extend(lo..=hi.min(MAX_CPUS - 1));
}
_ => {}
}
}
cpus
}
pub(crate) fn order(
mask: &[u16],
topology: &Topology,
strategy: PinStrategy,
out: &mut [u16],
) -> usize {
let m = mask.len().min(MAX_CPUS);
let n = out.len().min(m);
if strategy == PinStrategy::Pack {
out[..n].copy_from_slice(&mask[..n]);
return n;
}
let mut seen = [0u16; UNKNOWN + 1];
let mut rank = [u16::MAX; UNKNOWN + 1];
let mut groups = 0u16;
let mut keyed = [(0u32, 0u16); MAX_CPUS];
for (slot, &cpu) in keyed.iter_mut().zip(&mask[..m]) {
let g = topology.group(cpu);
if rank[g] == u16::MAX {
rank[g] = groups;
groups += 1;
}
*slot = ((u32::from(seen[g]) << 16) | u32::from(rank[g]), cpu);
seen[g] += 1;
}
keyed[..m].sort_unstable_by_key(|&(key, _)| key);
for (o, &(_, cpu)) in out[..n].iter_mut().zip(&keyed[..m]) {
*o = cpu;
}
n
}
#[cfg_attr(not(test), allow(dead_code))]
pub(crate) fn cpu_for(
mask: &[u16],
topology: &Topology,
strategy: PinStrategy,
k: usize,
) -> Option<u16> {
if k >= mask.len().min(MAX_CPUS) {
return None;
}
let mut out = [0u16; MAX_CPUS];
let n = order(mask, topology, strategy, &mut out[..=k]);
(n > k).then(|| out[k])
}
pub(crate) struct Placement {
cpus: [u16; MAX_CPUS],
len: usize,
}
impl Placement {
pub(crate) fn of(set: &CpuMask, strategy: PinStrategy, participants: usize) -> Placement {
let empty = Topology::default();
let topology = match strategy {
PinStrategy::Pack => &empty,
PinStrategy::Spread => host_topology(),
};
Placement::with_topology(set, strategy, participants, topology)
}
fn with_topology(
set: &CpuMask,
strategy: PinStrategy,
participants: usize,
topology: &Topology,
) -> Placement {
let mut mask = [0u16; MAX_CPUS];
let m = cpus(set, &mut mask);
let mut placement = Placement {
cpus: [0; MAX_CPUS],
len: 0,
};
let want = participants.min(MAX_CPUS);
placement.len = order(&mask[..m], topology, strategy, &mut placement.cpus[..want]);
placement
}
pub(crate) fn get(&self, k: usize) -> Option<usize> {
(k < self.len).then(|| usize::from(self.cpus[k]))
}
}
#[derive(Clone, Copy)]
struct Placed {
wide: CpuMask,
home: usize,
}
thread_local! {
static PLACED: core::cell::Cell<Option<Placed>> = const { core::cell::Cell::new(None) };
}
pub(crate) struct LeaderPin {
wide: CpuMask,
prev: Option<Placed>,
}
impl LeaderPin {
pub(crate) fn enter(wide: &CpuMask, home: usize) -> Option<LeaderPin> {
if !set_here_to(home) {
return None;
}
let prev = PLACED.with(|p| p.replace(Some(Placed { wide: *wide, home })));
Some(LeaderPin { wide: *wide, prev })
}
}
impl Drop for LeaderPin {
fn drop(&mut self) {
let _ = set_here(&self.wide);
PLACED.with(|p| p.set(self.prev));
}
}
pub(crate) type Home = usize;
pub(crate) fn home_of(placement: &Placement) -> Home {
placement.get(0).map_or(0, |cpu| cpu + 1)
}
pub(crate) fn home_pin(home: Home) -> Option<LeaderPin> {
let home = home.checked_sub(1)?;
let here = read_here()?;
if count(&here) < 2 || !contains(&here, home) {
return None;
}
if let Some(placed) = PLACED.with(|p| p.get()) {
if !subset(&here, &placed.wide) {
return None;
}
}
LeaderPin::enter(&here, home)
}
pub(crate) fn widened<R>(f: impl FnOnce() -> R) -> R {
struct Restore(usize);
impl Drop for Restore {
fn drop(&mut self) {
let _ = set_here_to(self.0);
}
}
let placed = PLACED.with(|p| p.get());
let _restore = placed.and_then(|p| set_here(&p.wide).then(|| Restore(p.home)));
f()
}
#[cfg_attr(not(feature = "parallel"), allow(dead_code))]
pub(crate) struct SavedMask {
mask: core::cell::UnsafeCell<core::mem::MaybeUninit<CpuMask>>,
pinned: AtomicBool,
}
unsafe impl Sync for SavedMask {}
#[cfg_attr(not(feature = "parallel"), allow(dead_code))]
impl SavedMask {
pub(crate) const fn new() -> Self {
SavedMask {
mask: core::cell::UnsafeCell::new(core::mem::MaybeUninit::uninit()),
pinned: AtomicBool::new(false),
}
}
pub(crate) fn save_if_unpinned(&self) {
if self.pinned.load(Ordering::Relaxed) {
return;
}
if let Some(mask) = read_here() {
unsafe { (*self.mask.get()).write(mask) };
self.pinned.store(true, Ordering::Relaxed);
}
}
pub(crate) fn restore(&self) {
if !self.pinned.load(Ordering::Relaxed) {
return;
}
let mask = unsafe { (*self.mask.get()).assume_init_ref() };
if set_here(mask) {
self.pinned.store(false, Ordering::Relaxed);
}
}
}
#[cfg(target_os = "linux")]
mod sys {
use core::mem;
const SETSIZE: usize = 8 * mem::size_of::<libc::cpu_set_t>();
#[derive(Clone, Copy)]
pub(crate) struct CpuMask {
set: libc::cpu_set_t,
}
#[cfg(test)]
pub(crate) fn mask_from(cpus: &[u16]) -> CpuMask {
unsafe {
let mut set = mem::zeroed::<libc::cpu_set_t>();
libc::CPU_ZERO(&mut set);
for &cpu in cpus {
libc::CPU_SET(usize::from(cpu), &mut set);
}
CpuMask { set }
}
}
pub(crate) fn read_here() -> Option<CpuMask> {
let mut set = unsafe { mem::zeroed::<libc::cpu_set_t>() };
let r = unsafe { libc::sched_getaffinity(0, mem::size_of::<libc::cpu_set_t>(), &mut set) };
(r == 0).then_some(CpuMask { set })
}
pub(crate) fn set_here(mask: &CpuMask) -> bool {
let r = unsafe { libc::sched_setaffinity(0, mem::size_of::<libc::cpu_set_t>(), &mask.set) };
r == 0
}
pub(crate) fn set_here_to(cpu: usize) -> bool {
if cpu >= SETSIZE {
return false;
}
unsafe {
let mut set = mem::zeroed::<libc::cpu_set_t>();
libc::CPU_ZERO(&mut set);
libc::CPU_SET(cpu, &mut set);
set_here(&CpuMask { set })
}
}
pub(crate) fn contains(mask: &CpuMask, cpu: usize) -> bool {
if cpu >= SETSIZE {
return false;
}
unsafe { libc::CPU_ISSET(cpu, &mask.set) }
}
pub(crate) fn subset(a: &CpuMask, b: &CpuMask) -> bool {
(0..SETSIZE).all(|cpu| {
!unsafe { libc::CPU_ISSET(cpu, &a.set) } || unsafe { libc::CPU_ISSET(cpu, &b.set) }
})
}
pub(crate) fn count(mask: &CpuMask) -> usize {
let n = unsafe { libc::CPU_COUNT(&mask.set) };
n.max(0) as usize
}
pub(crate) fn cpus(mask: &CpuMask, out: &mut [u16]) -> usize {
let mut n = 0;
for cpu in 0..SETSIZE.min(usize::from(u16::MAX)) {
if n == out.len() {
break;
}
if unsafe { libc::CPU_ISSET(cpu, &mask.set) } {
out[n] = cpu as u16;
n += 1;
}
}
n
}
}
#[cfg(not(target_os = "linux"))]
#[allow(dead_code)] mod sys {
#[derive(Clone, Copy)]
pub(crate) struct CpuMask(());
pub(crate) fn read_here() -> Option<CpuMask> {
None
}
pub(crate) fn set_here(_mask: &CpuMask) -> bool {
false
}
pub(crate) fn set_here_to(_cpu: usize) -> bool {
false
}
pub(crate) fn contains(_mask: &CpuMask, _cpu: usize) -> bool {
false
}
pub(crate) fn subset(_a: &CpuMask, _b: &CpuMask) -> bool {
false
}
pub(crate) fn count(_mask: &CpuMask) -> usize {
0
}
pub(crate) fn cpus(_mask: &CpuMask, _out: &mut [u16]) -> usize {
0
}
}
#[cfg(test)]
mod tests {
use super::*;
fn place(nodes: &[Vec<usize>], mask: &[u16], strategy: PinStrategy, n: usize) -> Vec<u16> {
let topology = Topology::from_nodes(nodes);
let mut out = vec![0u16; n];
let len = order(mask, &topology, strategy, &mut out);
out.truncate(len);
out
}
fn two_nodes() -> Vec<Vec<usize>> {
vec![vec![0, 1], vec![2, 3]]
}
#[cfg(target_os = "linux")]
#[test]
fn placement_takes_the_strategy_on_a_two_node_machine() {
use super::sys::mask_from;
let set = mask_from(&[0, 1, 2, 3]);
let topology = Topology::from_nodes(&two_nodes());
let placed = |strategy| {
let placement = Placement::with_topology(&set, strategy, 2, &topology);
(0..2).filter_map(|k| placement.get(k)).collect::<Vec<_>>()
};
assert_eq!(placed(PinStrategy::Spread), vec![0, 2]);
assert_eq!(placed(PinStrategy::Pack), vec![0, 1]);
let wide = |strategy| {
let placement = Placement::with_topology(&set, strategy, 4, &topology);
(0..4).filter_map(|k| placement.get(k)).collect::<Vec<_>>()
};
assert_eq!(wide(PinStrategy::Spread), vec![0, 2, 1, 3]);
assert_eq!(wide(PinStrategy::Pack), vec![0, 1, 2, 3]);
}
fn four_nodes() -> Vec<Vec<usize>> {
vec![vec![0, 1], vec![2, 3], vec![4, 5], vec![6, 7]]
}
#[test]
fn spread_alternates_the_nodes_of_a_two_node_mask() {
let got = place(&two_nodes(), &[0, 1, 2, 3], PinStrategy::Spread, 4);
assert_eq!(got, [0, 2, 1, 3]);
}
#[test]
fn pack_is_the_mask_in_ascending_order() {
let got = place(&two_nodes(), &[0, 1, 2, 3], PinStrategy::Pack, 4);
assert_eq!(got, [0, 1, 2, 3]);
let got = place(&two_nodes(), &[0, 1, 2, 3], PinStrategy::Pack, 2);
assert_eq!(got, [0, 1], "a narrow packed team sits on node 0");
}
#[test]
fn a_narrow_spread_team_covers_both_nodes() {
let got = place(&two_nodes(), &[0, 1, 2, 3], PinStrategy::Spread, 2);
assert_eq!(got, [0, 2]);
}
#[test]
fn a_team_of_one_on_a_four_node_mask_gets_one_cpu() {
let mask: Vec<u16> = (0..8).collect();
for strategy in [PinStrategy::Spread, PinStrategy::Pack] {
assert_eq!(
place(&four_nodes(), &mask, strategy, 1),
[0],
"{strategy:?}"
);
}
}
#[test]
fn spread_never_leaves_the_mask() {
let got = place(&four_nodes(), &[1, 2, 3, 5], PinStrategy::Spread, 4);
assert_eq!(got, [1, 2, 5, 3]);
}
#[test]
fn uneven_nodes_drop_out_of_the_rotation_when_empty() {
let nodes = vec![vec![0, 1, 2], vec![3]];
let got = place(&nodes, &[0, 1, 2, 3], PinStrategy::Spread, 4);
assert_eq!(got, [0, 3, 1, 2]);
}
#[test]
fn nodes_are_visited_in_the_order_of_their_lowest_cpu() {
let nodes = vec![vec![4, 5], vec![0, 1]];
let got = place(&nodes, &[0, 1, 4, 5], PinStrategy::Spread, 4);
assert_eq!(got, [0, 4, 1, 5]);
let nodes = vec![vec![0, 2], vec![1, 3]];
let got = place(&nodes, &[0, 1, 2, 3], PinStrategy::Spread, 4);
assert_eq!(got, [0, 1, 2, 3]);
}
#[test]
fn without_a_topology_spread_places_as_pack() {
let mask: Vec<u16> = (0..6).collect();
let got = place(&[], &mask, PinStrategy::Spread, 6);
assert_eq!(got, mask, "every CPU in one unknown group");
let got = place(&[vec![0, 1]], &[0, 1, 8, 9], PinStrategy::Spread, 4);
assert_eq!(got, [0, 8, 1, 9]);
}
#[test]
fn a_placement_is_never_longer_than_the_mask() {
let got = place(&two_nodes(), &[0, 2], PinStrategy::Spread, 3);
assert_eq!(got, [0, 2], "no wrapping");
}
#[test]
fn spread_covers_as_many_nodes_as_the_team_allows() {
let two = vec![vec![0, 1, 2, 3], vec![4, 5, 6, 7]];
for nodes in [two, four_nodes()] {
let node_of = |cpu: u16| nodes.iter().position(|n| n.contains(&usize::from(cpu)));
for bits in 1u32..256 {
let mask: Vec<u16> = (0..8).filter(|c| bits & (1 << c) != 0).collect();
let mut spanned: Vec<_> = mask.iter().map(|&c| node_of(c)).collect();
spanned.sort();
spanned.dedup();
for n in 1..=mask.len() {
let got = place(&nodes, &mask, PinStrategy::Spread, n);
assert_eq!(got.len(), n);
assert_eq!(got[0], mask[0], "the leader is on S[0]");
let mut sorted = got.clone();
sorted.sort();
sorted.dedup();
assert_eq!(sorted.len(), n, "no CPU twice: {got:?}");
assert!(got.iter().all(|c| mask.contains(c)), "inside S");
let mut covered: Vec<_> = got.iter().map(|&c| node_of(c)).collect();
covered.sort();
covered.dedup();
assert_eq!(
covered.len(),
n.min(spanned.len()),
"mask {mask:?}, n {n}: {got:?}"
);
}
}
}
}
#[test]
fn on_a_one_node_mask_spread_and_pack_place_identically() {
let cases: [(Vec<Vec<usize>>, Vec<u16>); 3] = [
(four_nodes(), vec![2, 3]),
(vec![(0..8).collect()], (0..8).collect()),
(vec![], vec![1, 3, 4, 6]),
];
for (nodes, mask) in cases {
let topology = Topology::from_nodes(&nodes);
for k in 0..=mask.len() {
let spread = cpu_for(&mask, &topology, PinStrategy::Spread, k);
let pack = cpu_for(&mask, &topology, PinStrategy::Pack, k);
assert_eq!(spread, pack, "mask {mask:?}, k {k}");
assert_eq!(pack, mask.get(k).copied(), "Pack is nth(S, k)");
}
}
}
#[test]
fn cpu_for_is_entry_k_of_the_order() {
let topology = Topology::from_nodes(&two_nodes());
let mask = [0, 1, 2, 3];
let got: Vec<_> = (0..5)
.map(|k| cpu_for(&mask, &topology, PinStrategy::Spread, k))
.collect();
assert_eq!(got, [Some(0), Some(2), Some(1), Some(3), None]);
}
#[test]
fn a_narrow_spread_team_covers_all_four_nodes() {
let mask: Vec<u16> = (0..8).collect();
let got = place(&four_nodes(), &mask, PinStrategy::Spread, 4);
assert_eq!(got, [0, 2, 4, 6], "one CPU on each of the four nodes");
}
#[test]
fn sysfs_cpu_lists_parse() {
assert_eq!(parse_cpulist("0-3,8,10-11\n"), [0, 1, 2, 3, 8, 10, 11]);
assert_eq!(parse_cpulist(""), Vec::<usize>::new());
assert_eq!(parse_cpulist("5"), [5]);
assert_eq!(
parse_cpulist("x,2,3-1"),
[2],
"malformed pieces are skipped"
);
}
#[test]
fn a_refused_second_placement_keeps_the_recorded_set() {
let Some(wide) = read_here() else {
return; };
let width = count(&wide);
if width < 2 {
return; }
let home = Placement::of(&wide, PinStrategy::Pack, 1)
.get(0)
.expect("a CPU in a non-empty mask");
let Some(first) = LeaderPin::enter(&wide, home) else {
eprintln!("skipped: the kernel refused to pin this thread");
return;
};
assert_eq!(
available(),
width,
"the recorded set is the one the scope started in"
);
assert!(
home_pin(home + 1).is_none(),
"a second placement in the same scope is refused"
);
assert_eq!(
available(),
width,
"the refused placement must not overwrite the recorded set"
);
drop(first);
assert_eq!(
available(),
width,
"the set comes back when the placement ends"
);
}
#[cfg(target_os = "linux")]
#[test]
fn the_cpu_count_is_memoised_until_this_crate_moves_the_thread() {
let whole = match read_here() {
Some(mask) if count(&mask) >= 2 => mask,
_ => {
eprintln!("skipped: need at least 2 CPUs in the calling thread mask");
return;
}
};
let wide = count(&whole);
assert_eq!(available(), wide, "the first read answers the mask");
let mut first = [0u16; 1];
assert_eq!(cpus(&whole, &mut first), 1, "the mask holds a CPU");
let one = sys::mask_from(&first[..]);
assert!(sys::set_here(&one), "restricting the calling thread");
assert_eq!(
available(),
wide,
"no kernel read on the next mint: the memo answers"
);
assert!(set_here(&one), "a mask change this crate makes");
assert_eq!(available(), 1, "which forgets the memo");
assert!(set_here(&whole), "restoring the calling thread mask");
assert_eq!(available(), wide);
}
}