use std::sync::OnceLock;
const DEFAULT_LOCAL_DISTANCE: u8 = 10;
const DEFAULT_REMOTE_DISTANCE: u8 = 20;
#[derive(Clone, Debug)]
pub struct NumaTopology {
pub num_nodes: u32,
pub node_of_cpu: Vec<u32>,
pub distances: Vec<Vec<u8>>,
pub cluster_size_log2: u8,
pub cluster_source: ClusterSource,
pub source: NumaSource,
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub enum NumaSource {
LinuxSysfs,
WindowsGlpiEx,
Fallback,
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub enum ClusterSource {
None,
AmdCpuidCcx,
IntelCpuidModule,
ArmSysfsCluster,
AppleSysctlPerflevel,
}
fn logical_threads() -> usize {
std::thread::available_parallelism()
.map(std::num::NonZeroUsize::get)
.unwrap_or(1)
}
impl NumaTopology {
pub fn fallback() -> Self {
let n = logical_threads().max(1);
Self {
num_nodes: 1,
node_of_cpu: vec![0; n],
distances: vec![vec![DEFAULT_LOCAL_DISTANCE]],
cluster_size_log2: 0,
cluster_source: ClusterSource::None,
source: NumaSource::Fallback,
}
}
pub fn detect() -> Self {
let mut topo = primary_detect().unwrap_or_else(Self::fallback);
let (size_log2, src) = detect_cluster_size_log2();
topo.cluster_size_log2 = size_log2;
topo.cluster_source = src;
if topo.distances.len() as u32 != topo.num_nodes
|| topo
.distances
.iter()
.any(|row| row.len() as u32 != topo.num_nodes)
{
topo.distances = build_uniform_distance_matrix(topo.num_nodes);
}
topo
}
#[inline]
pub fn distance(&self, a: u32, b: u32) -> u8 {
let (i, j) = (a as usize, b as usize);
if i < self.distances.len() && j < self.distances[i].len() {
self.distances[i][j]
} else {
DEFAULT_LOCAL_DISTANCE
}
}
#[inline]
pub fn is_multi_node(&self) -> bool {
self.num_nodes >= 2
}
pub fn cpus_in_node(&self, node: u32) -> Vec<u32> {
self.node_of_cpu
.iter()
.enumerate()
.filter_map(|(i, &n)| if n == node { Some(i as u32) } else { None })
.collect()
}
}
pub fn numa_topology() -> &'static NumaTopology {
static CACHE: OnceLock<NumaTopology> = OnceLock::new();
CACHE.get_or_init(NumaTopology::detect)
}
fn build_uniform_distance_matrix(num_nodes: u32) -> Vec<Vec<u8>> {
let n = num_nodes.max(1) as usize;
let mut m = vec![vec![DEFAULT_REMOTE_DISTANCE; n]; n];
for (i, row) in m.iter_mut().enumerate() {
row[i] = DEFAULT_LOCAL_DISTANCE;
}
m
}
fn primary_detect() -> Option<NumaTopology> {
#[cfg(target_os = "linux")]
{
detect_linux()
}
#[cfg(target_os = "windows")]
{
detect_windows()
}
#[cfg(not(any(target_os = "linux", target_os = "windows")))]
{
None
}
}
#[cfg(target_os = "linux")]
fn detect_linux() -> Option<NumaTopology> {
use std::fs;
use std::path::Path;
let root = Path::new("/sys/devices/system/node");
let entries = fs::read_dir(root).ok()?;
let mut nodes: Vec<u32> = Vec::new();
for entry in entries.flatten() {
let name = entry.file_name();
let name = name.to_string_lossy();
let Some(rest) = name.strip_prefix("node") else { continue };
match rest.parse::<u32>() {
Ok(id) => nodes.push(id),
Err(e) => notice!(
"flynnel numa: {root}/node{rest} is not a node id ({e}), so that entry \
contributes no node",
root = root.display()
),
}
}
if nodes.is_empty() {
return None;
}
nodes.sort_unstable();
let lt = logical_threads().max(1);
let mut node_of_cpu = vec![0u32; lt];
for &node in &nodes {
let cpulist_path = root.join(format!("node{node}")).join("cpulist");
let Ok(s) = fs::read_to_string(&cpulist_path) else { continue };
for cpu in parse_cpulist(s.trim()) {
if (cpu as usize) < lt {
node_of_cpu[cpu as usize] = node;
}
}
}
let num_nodes = nodes.iter().max().copied().unwrap_or(0) + 1;
let mut distances = build_uniform_distance_matrix(num_nodes);
for &node in &nodes {
let dist_path = root.join(format!("node{node}")).join("distance");
let Ok(s) = fs::read_to_string(&dist_path) else { continue };
let row: Vec<u8> = s
.split_ascii_whitespace()
.filter_map(|tok| tok.parse::<u8>().ok())
.collect();
if row.len() as u32 >= num_nodes {
distances[node as usize] = row[..num_nodes as usize].to_vec();
}
}
Some(NumaTopology {
num_nodes,
node_of_cpu,
distances,
cluster_size_log2: 0,
cluster_source: ClusterSource::None,
source: NumaSource::LinuxSysfs,
})
}
#[cfg(target_os = "linux")]
fn parse_cpulist(s: &str) -> Vec<u32> {
let mut out = Vec::new();
for token in s.split(',') {
let token = token.trim();
if token.is_empty() {
continue;
}
if let Some((lo, hi)) = token.split_once('-') {
if let (Ok(lo), Ok(hi)) = (lo.parse::<u32>(), hi.parse::<u32>()) {
for cpu in lo..=hi {
out.push(cpu);
}
}
} else if let Ok(cpu) = token.parse::<u32>() {
out.push(cpu);
}
}
out
}
#[cfg(target_os = "windows")]
#[allow(non_camel_case_types, non_snake_case, dead_code, clippy::upper_case_acronyms)]
mod win {
pub type KAFFINITY = usize;
pub const RELATION_NUMA_NODE: u32 = 1;
pub const ERROR_INSUFFICIENT_BUFFER: u32 = 122;
#[repr(C)]
#[derive(Copy, Clone)]
pub struct GROUP_AFFINITY {
pub mask: KAFFINITY,
pub group: u16,
pub reserved: [u16; 3],
}
#[repr(C)]
#[derive(Copy, Clone)]
pub struct SlpiExHeader {
pub relationship: u32,
pub size: u32,
}
#[repr(C)]
#[derive(Copy, Clone)]
pub struct NumaNodeRelationship {
pub node_number: u32,
pub reserved: [u8; 20],
pub group_mask: GROUP_AFFINITY,
}
#[link(name = "kernel32")]
unsafe extern "system" {
pub fn GetLogicalProcessorInformationEx(
relationship_type: u32,
buffer: *mut u8,
returned_length: *mut u32,
) -> i32;
pub fn GetLastError() -> u32;
}
}
#[cfg(target_os = "windows")]
fn detect_windows() -> Option<NumaTopology> {
let mut required: u32 = 0;
let ok = unsafe {
win::GetLogicalProcessorInformationEx(
win::RELATION_NUMA_NODE,
core::ptr::null_mut(),
&mut required,
)
};
if ok != 0 || required == 0 {
return None;
}
if unsafe { win::GetLastError() } != win::ERROR_INSUFFICIENT_BUFFER {
return None;
}
let mut buf: Vec<u8> = vec![0u8; required as usize];
let mut len = required;
let ok2 = unsafe {
win::GetLogicalProcessorInformationEx(
win::RELATION_NUMA_NODE,
buf.as_mut_ptr(),
&mut len,
)
};
if ok2 == 0 {
return None;
}
let lt = logical_threads().max(1);
let mut node_of_cpu = vec![0u32; lt];
let mut max_node: u32 = 0;
let mut node_seen: Vec<bool> = Vec::new();
let mut offset: usize = 0;
while offset + core::mem::size_of::<win::SlpiExHeader>() <= len as usize {
let header: win::SlpiExHeader = unsafe {
core::ptr::read_unaligned(buf.as_ptr().add(offset) as *const _)
};
let entry_size = header.size as usize;
if entry_size == 0 || offset + entry_size > len as usize {
break;
}
if header.relationship == win::RELATION_NUMA_NODE {
let body_offset = offset + core::mem::size_of::<win::SlpiExHeader>();
if body_offset + core::mem::size_of::<win::NumaNodeRelationship>() <= len as usize {
let body: win::NumaNodeRelationship = unsafe {
core::ptr::read_unaligned(
buf.as_ptr().add(body_offset) as *const _,
)
};
let node = body.node_number;
if node > max_node {
max_node = node;
}
if (node as usize) >= node_seen.len() {
node_seen.resize(node as usize + 1, false);
}
node_seen[node as usize] = true;
let mask = body.group_mask.mask;
let group = body.group_mask.group;
let base = (group as usize) << 6;
for bit in 0..(core::mem::size_of::<win::KAFFINITY>() << 3) {
if (mask >> bit) & 1 == 1 {
let cpu = base + bit;
if cpu < lt {
node_of_cpu[cpu] = node;
}
}
}
}
}
offset += entry_size;
}
let num_nodes = node_seen.iter().filter(|&&seen| seen).count() as u32;
let num_nodes = num_nodes.max(1);
let distances = build_uniform_distance_matrix(num_nodes);
Some(NumaTopology {
num_nodes,
node_of_cpu,
distances,
cluster_size_log2: 0,
cluster_source: ClusterSource::None,
source: NumaSource::WindowsGlpiEx,
})
}
fn detect_cluster_size_log2() -> (u8, ClusterSource) {
#[cfg(target_arch = "x86_64")]
{
if is_amd_cpu() {
let n = unsafe { amd_ccx_threads_log2() };
if n > 0 {
return (n, ClusterSource::AmdCpuidCcx);
}
}
if is_intel_cpu() {
let n = unsafe { intel_module_threads_log2() };
if n > 0 {
return (n, ClusterSource::IntelCpuidModule);
}
}
(0, ClusterSource::None)
}
#[cfg(target_arch = "aarch64")]
{
#[cfg(target_os = "linux")]
{
let n = arm_linux_cluster_size_log2();
if n > 0 {
return (n, ClusterSource::ArmSysfsCluster);
}
}
#[cfg(target_os = "macos")]
{
let n = apple_perflevel0_size_log2();
if n > 0 {
return (n, ClusterSource::AppleSysctlPerflevel);
}
}
(0, ClusterSource::None)
}
#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
{
(0, ClusterSource::None)
}
}
#[cfg(target_arch = "x86_64")]
fn is_amd_cpu() -> bool {
use std::arch::x86_64::__cpuid;
let r = __cpuid(0);
r.ebx == 0x6874_7541 && r.edx == 0x6974_6e65 && r.ecx == 0x444d_4163
}
#[cfg(target_arch = "x86_64")]
fn is_intel_cpu() -> bool {
use std::arch::x86_64::__cpuid;
let r = __cpuid(0);
r.ebx == 0x756e_6547 && r.edx == 0x4965_6e69 && r.ecx == 0x6c65_746e
}
#[cfg(target_arch = "x86_64")]
unsafe fn amd_ccx_threads_log2() -> u8 {
use std::arch::x86_64::__cpuid_count;
let max_ext = __cpuid_count(0x8000_0000, 0).eax;
if max_ext < 0x8000_001D {
return 0;
}
for sub in 0u32..16 {
let r = __cpuid_count(0x8000_001D, sub);
let cache_type = r.eax & 0x1F;
if cache_type == 0 {
break;
}
let level = (r.eax >> 5) & 0x07;
if level == 3 {
let num_sharing = ((r.eax >> 14) & 0x0FFF) + 1;
return log2_u32(num_sharing);
}
}
0
}
#[cfg(target_arch = "x86_64")]
unsafe fn intel_module_threads_log2() -> u8 {
use std::arch::x86_64::__cpuid_count;
let max_basic = __cpuid_count(0, 0).eax;
if max_basic < 0x1F {
return 0;
}
const MODULE_DOMAIN: u32 = 3;
for sub in 0u32..32 {
let r = __cpuid_count(0x1F, sub);
if r.ebx & 0xFFFF == 0 {
break;
}
let domain_type = (r.ecx >> 8) & 0xFF;
if domain_type == MODULE_DOMAIN {
let lp_count = r.ebx & 0xFFFF;
return log2_u32(lp_count);
}
}
0
}
#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
fn arm_linux_cluster_size_log2() -> u8 {
use std::fs;
let cpu0_cluster_id =
fs::read_to_string("/sys/devices/system/cpu/cpu0/topology/cluster_id")
.ok()
.and_then(|s| s.trim().parse::<i32>().ok());
let Some(cpu0_id) = cpu0_cluster_id else { return 0 };
if cpu0_id < 0 {
return 0;
}
let mut count: u32 = 0;
let lt = logical_threads().max(1) as u32;
for cpu in 0..lt {
let path = format!("/sys/devices/system/cpu/cpu{cpu}/topology/cluster_id");
if let Ok(s) = fs::read_to_string(&path) {
if let Ok(id) = s.trim().parse::<i32>() {
if id == cpu0_id {
count += 1;
}
}
}
}
log2_u32(count.max(1))
}
#[cfg(all(target_arch = "aarch64", target_os = "macos"))]
fn apple_perflevel0_size_log2() -> u8 {
unsafe extern "C" {
fn sysctlbyname(
name: *const core::ffi::c_char,
oldp: *mut core::ffi::c_void,
oldlenp: *mut usize,
newp: *const core::ffi::c_void,
newlen: usize,
) -> core::ffi::c_int;
}
let mut value: i64 = 0;
let mut size: usize = core::mem::size_of::<i64>();
let rc = unsafe {
sysctlbyname(
c"hw.perflevel0.physicalcpu".as_ptr(),
&mut value as *mut i64 as *mut core::ffi::c_void,
&mut size,
core::ptr::null(),
0,
)
};
if rc != 0 || value <= 0 {
return 0;
}
log2_u32(value as u32)
}
fn log2_u32(x: u32) -> u8 {
if x <= 1 {
0
} else {
(31 - x.leading_zeros()) as u8
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn fallback_is_single_node() {
let f = NumaTopology::fallback();
assert_eq!(f.num_nodes, 1);
assert_eq!(f.distances, vec![vec![DEFAULT_LOCAL_DISTANCE]]);
assert_eq!(f.source, NumaSource::Fallback);
assert_eq!(f.cluster_size_log2, 0);
assert_eq!(f.cluster_source, ClusterSource::None);
for &n in &f.node_of_cpu {
assert_eq!(n, 0);
}
}
#[test]
fn fallback_node_of_cpu_matches_logical_threads() {
let f = NumaTopology::fallback();
assert_eq!(f.node_of_cpu.len(), logical_threads().max(1));
}
#[test]
fn numa_topology_is_cached() {
let a = numa_topology() as *const _;
let b = numa_topology() as *const _;
assert_eq!(a, b);
}
#[test]
fn detect_produces_sane_snapshot() {
let t = NumaTopology::detect();
assert!(t.num_nodes >= 1, "num_nodes must be >= 1");
assert_eq!(
t.distances.len() as u32,
t.num_nodes,
"distance matrix must have num_nodes rows"
);
for row in &t.distances {
assert_eq!(
row.len() as u32,
t.num_nodes,
"distance matrix must be square"
);
}
for i in 0..t.num_nodes as usize {
assert_eq!(
t.distances[i][i],
DEFAULT_LOCAL_DISTANCE,
"diagonal entry [{i}][{i}] must be 10"
);
}
let lt = logical_threads().max(1);
assert_eq!(t.node_of_cpu.len(), lt);
for &n in &t.node_of_cpu {
assert!(
n < t.num_nodes,
"node id {n} out of range (num_nodes = {})",
t.num_nodes
);
}
}
#[test]
fn distance_accessor_handles_oob() {
let t = NumaTopology::fallback();
assert_eq!(t.distance(99, 0), DEFAULT_LOCAL_DISTANCE);
assert_eq!(t.distance(0, 99), DEFAULT_LOCAL_DISTANCE);
}
#[test]
fn cpus_in_node_round_trips_against_node_of_cpu() {
let t = numa_topology();
for node in 0..t.num_nodes {
for cpu in t.cpus_in_node(node) {
assert_eq!(
t.node_of_cpu[cpu as usize], node,
"cpu {cpu} reported in node {node} but node_of_cpu says {}",
t.node_of_cpu[cpu as usize]
);
}
}
}
#[test]
fn is_multi_node_matches_num_nodes() {
assert!(!NumaTopology::fallback().is_multi_node());
let t = numa_topology();
assert_eq!(t.is_multi_node(), t.num_nodes >= 2);
}
#[test]
fn parse_cpulist_handles_ranges_and_singletons() {
#[cfg(target_os = "linux")]
{
assert_eq!(super::parse_cpulist("0-3"), vec![0, 1, 2, 3]);
assert_eq!(super::parse_cpulist("0,2,4"), vec![0, 2, 4]);
assert_eq!(super::parse_cpulist("0-1,3,5-6"), vec![0, 1, 3, 5, 6]);
assert_eq!(super::parse_cpulist(""), Vec::<u32>::new());
}
}
#[test]
fn log2_u32_matches_intrinsic() {
for k in 0u32..16 {
assert_eq!(log2_u32(1u32 << k) as u32, k, "log2({})", 1u32 << k);
}
assert_eq!(log2_u32(0), 0);
assert_eq!(log2_u32(1), 0);
}
#[test]
fn cluster_detection_returns_sane_value() {
let (n, src) = detect_cluster_size_log2();
assert!(n <= 7, "cluster_size_log2 {n} unreasonably large");
if n > 0 {
assert_ne!(src, ClusterSource::None);
} else {
assert_eq!(src, ClusterSource::None);
}
}
#[test]
fn cluster_size_is_recorded_on_detect_snapshot() {
let t = NumaTopology::detect();
assert!(t.cluster_size_log2 <= 7);
if t.cluster_size_log2 > 0 {
assert_ne!(t.cluster_source, ClusterSource::None);
} else {
assert_eq!(t.cluster_source, ClusterSource::None);
}
}
#[test]
fn build_uniform_distance_matrix_diagonal_is_local() {
for n in 1u32..6 {
let m = build_uniform_distance_matrix(n);
assert_eq!(m.len() as u32, n);
for (i, row) in m.iter().enumerate() {
assert_eq!(row.len() as u32, n);
assert_eq!(
row[i], DEFAULT_LOCAL_DISTANCE,
"[{i}][{i}] not local for n={n}"
);
for (j, &cell) in row.iter().enumerate() {
if i != j {
assert_eq!(cell, DEFAULT_REMOTE_DISTANCE);
}
}
}
}
}
#[cfg(target_arch = "x86_64")]
#[test]
fn vendor_probes_are_mutually_exclusive() {
assert!(!(is_amd_cpu() && is_intel_cpu()));
}
}