use crate::error::IronError;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NumaNode {
pub id: u32,
pub cpus: Option<Vec<u32>>,
pub memory_total_bytes: Option<u64>,
pub memory_free_bytes: Option<u64>,
pub memory_used_bytes: Option<u64>,
pub hugepages_total: Option<u64>,
pub hugepages_free: Option<u64>,
pub pci_devices: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NumaDistanceMatrix {
pub size: usize,
pub distances: Vec<u32>,
}
impl NumaDistanceMatrix {
pub fn distance(&self, from: usize, to: usize) -> Option<u32> {
if from < self.size && to < self.size {
Some(self.distances[from * self.size + to])
} else {
None
}
}
pub fn is_numa(&self) -> bool {
self.size > 1 && self.distances.iter().any(|&d| d != 10)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NumaSummary {
pub node_count: usize,
pub total_cpus: Option<usize>,
pub total_memory_bytes: Option<u64>,
pub is_numa: bool,
pub max_distance: Option<u32>,
pub memory_imbalance_ratio: Option<f64>,
pub cpu_imbalance_ratio: Option<f64>,
}
pub struct NumaMonitor {
nodes: Vec<NumaNode>,
distance_matrix: Option<NumaDistanceMatrix>,
}
impl NumaMonitor {
pub fn new() -> Result<Self, IronError> {
let mut monitor = Self {
nodes: Vec::new(),
distance_matrix: None,
};
monitor.refresh()?;
Ok(monitor)
}
pub fn refresh(&mut self) -> Result<(), IronError> {
self.nodes.clear();
self.distance_matrix = None;
#[cfg(target_os = "linux")]
self.refresh_linux()?;
#[cfg(target_os = "windows")]
self.refresh_windows()?;
#[cfg(target_os = "macos")]
self.refresh_macos()?;
Ok(())
}
pub fn nodes(&self) -> &[NumaNode] {
&self.nodes
}
pub fn distance_matrix(&self) -> Option<&NumaDistanceMatrix> {
self.distance_matrix.as_ref()
}
pub fn summary(&self) -> NumaSummary {
let total_cpus: Option<usize> = self
.nodes
.iter()
.map(|n| n.cpus.as_ref().map(Vec::len))
.sum();
let total_memory: Option<u64> = self.nodes.iter().map(|n| n.memory_total_bytes).sum();
let is_numa =
self.nodes.len() > 1 || self.distance_matrix.as_ref().is_some_and(|d| d.is_numa());
let max_distance = self
.distance_matrix
.as_ref()
.and_then(|d| d.distances.iter().copied().max());
let mem_vals: Vec<u64> = self
.nodes
.iter()
.filter_map(|n| n.memory_total_bytes)
.collect();
let memory_imbalance = match (mem_vals.iter().max(), mem_vals.iter().min()) {
(Some(&max), Some(&min)) if mem_vals.len() >= 2 && min > 0 => {
Some(max as f64 / min as f64)
}
_ => None,
};
let cpu_counts: Option<Vec<usize>> = self
.nodes
.iter()
.map(|n| n.cpus.as_ref().map(Vec::len))
.collect();
let cpu_imbalance = cpu_counts.and_then(|counts| {
let vals: Vec<usize> = counts.into_iter().filter(|&c| c > 0).collect();
match (vals.iter().max(), vals.iter().min()) {
(Some(&max), Some(&min)) if vals.len() >= 2 && min > 0 => {
Some(max as f64 / min as f64)
}
_ => None,
}
});
NumaSummary {
node_count: self.nodes.len(),
total_cpus,
total_memory_bytes: total_memory,
is_numa,
max_distance,
memory_imbalance_ratio: memory_imbalance,
cpu_imbalance_ratio: cpu_imbalance,
}
}
pub fn cpu_to_node(&self, cpu_id: u32) -> Option<u32> {
self.nodes
.iter()
.find(|n| n.cpus.as_ref().is_some_and(|c| c.contains(&cpu_id)))
.map(|n| n.id)
}
pub fn node(&self, id: u32) -> Option<&NumaNode> {
self.nodes.iter().find(|n| n.id == id)
}
#[cfg(target_os = "linux")]
fn refresh_linux(&mut self) -> Result<(), IronError> {
let node_base = std::path::Path::new("/sys/devices/system/node");
if !node_base.exists() {
self.single_node_fallback(Self::online_cpus());
return Ok(());
}
let entries = std::fs::read_dir(node_base)
.map_err(|e| IronError::System(format!("cannot list {}: {e}", node_base.display())))?;
let mut node_ids: Vec<u32> = Vec::new();
for entry in entries.flatten() {
let name = entry.file_name().to_string_lossy().to_string();
if let Some(id_str) = name.strip_prefix("node") {
if let Ok(id) = id_str.parse::<u32>() {
node_ids.push(id);
}
}
}
if node_ids.is_empty() {
return Err(IronError::System(format!(
"{} lists no nodeN entries",
node_base.display()
)));
}
node_ids.sort();
for id in &node_ids {
let node_path = node_base.join(format!("node{}", id));
let cpus = std::fs::read_to_string(node_path.join("cpulist"))
.ok()
.map(|s| Self::parse_cpu_list(s.trim()));
let (total, free, used) = std::fs::read_to_string(node_path.join("meminfo"))
.map(|s| Self::parse_node_meminfo(&s))
.unwrap_or((None, None, None));
let hp_total =
std::fs::read_to_string(node_path.join("hugepages/hugepages-2048kB/nr_hugepages"))
.ok()
.and_then(|s| s.trim().parse().ok());
let hp_free = std::fs::read_to_string(
node_path.join("hugepages/hugepages-2048kB/free_hugepages"),
)
.ok()
.and_then(|s| s.trim().parse().ok());
let mut pci_devs = Vec::new();
if let Ok(entries) = std::fs::read_dir("/sys/bus/pci/devices") {
for entry in entries.flatten() {
let numa_node_path = entry.path().join("numa_node");
if let Ok(node_str) = std::fs::read_to_string(&numa_node_path) {
if let Ok(nid) = node_str.trim().parse::<i32>() {
if nid == *id as i32 {
pci_devs.push(entry.file_name().to_string_lossy().to_string());
}
}
}
}
}
self.nodes.push(NumaNode {
id: *id,
cpus,
memory_total_bytes: total,
memory_free_bytes: free,
memory_used_bytes: used,
hugepages_total: hp_total,
hugepages_free: hp_free,
pci_devices: pci_devs,
});
}
if let Some(first_node) = node_ids.first() {
let dist_path = node_base.join(format!("node{}/distance", first_node));
if let Ok(_text) = std::fs::read_to_string(&dist_path) {
let size = node_ids.len();
let mut distances = Vec::with_capacity(size * size);
for nid in &node_ids {
let p = node_base.join(format!("node{}/distance", nid));
if let Ok(line) = std::fs::read_to_string(&p) {
for val in line.split_whitespace() {
if let Ok(d) = val.parse::<u32>() {
distances.push(d);
}
}
}
}
if distances.len() == size * size {
self.distance_matrix = Some(NumaDistanceMatrix { size, distances });
}
}
}
Ok(())
}
#[cfg(target_os = "linux")]
fn online_cpus() -> Option<Vec<u32>> {
std::fs::read_to_string("/sys/devices/system/cpu/online")
.ok()
.map(|s| Self::parse_cpu_list(s.trim()))
}
#[cfg(target_os = "linux")]
fn parse_cpu_list(s: &str) -> Vec<u32> {
let mut result = Vec::new();
for part in s.split(',') {
let part = part.trim();
if let Some((start, end)) = part.split_once('-') {
if let (Ok(s), Ok(e)) = (start.parse::<u32>(), end.parse::<u32>()) {
result.extend(s..=e);
}
} else if let Ok(n) = part.parse::<u32>() {
result.push(n);
}
}
result
}
#[cfg(target_os = "linux")]
fn parse_node_meminfo(text: &str) -> (Option<u64>, Option<u64>, Option<u64>) {
let mut total = None;
let mut free = None;
for line in text.lines() {
if line.contains("MemTotal:") {
total = Self::extract_kb(line).map(|kb| kb * 1024);
} else if line.contains("MemFree:") {
free = Self::extract_kb(line).map(|kb| kb * 1024);
}
}
let used = match (total, free) {
(Some(t), Some(f)) => Some(t.saturating_sub(f)),
_ => None,
};
(total, free, used)
}
#[cfg(target_os = "linux")]
fn extract_kb(line: &str) -> Option<u64> {
let (_, value) = line.split_once(':')?;
value.split_whitespace().next()?.parse().ok()
}
#[cfg(target_os = "windows")]
fn refresh_windows(&mut self) -> Result<(), IronError> {
use windows::Win32::System::Threading::{
GetNumaAvailableMemoryNodeEx, GetNumaHighestNodeNumber,
};
let mut highest: u32 = 0;
unsafe { GetNumaHighestNodeNumber(&mut highest) }
.map_err(|e| IronError::System(format!("GetNumaHighestNodeNumber failed: {e}")))?;
let node_count = highest as u64 + 1;
let total_bytes = crate::platform::windows::read_memory_stats()
.ok()
.map(|m| m.ram.total.saturating_mul(1024));
let logical = std::thread::available_parallelism()
.ok()
.map(|p| p.get() as u32);
for id in 0..=highest {
let mut available: u64 = 0;
let mut free = None;
if unsafe { GetNumaAvailableMemoryNodeEx(id as u16, &mut available) }.is_ok() {
free = Some(available);
}
let (memory_total_bytes, cpus) = if node_count == 1 {
(total_bytes, logical.map(|n| (0..n).collect::<Vec<u32>>()))
} else {
(None, None)
};
self.nodes.push(NumaNode {
id,
cpus,
memory_total_bytes,
memory_free_bytes: free,
memory_used_bytes: match (memory_total_bytes, free) {
(Some(total), Some(free)) => Some(total.saturating_sub(free)),
_ => None,
},
hugepages_total: None,
hugepages_free: None,
pci_devices: Vec::new(),
});
}
Ok(())
}
#[cfg(target_os = "macos")]
fn refresh_macos(&mut self) -> Result<(), IronError> {
let mut total_mem = None;
let mut total_cpus: Option<u32> = None;
if let Ok(output) = std::process::Command::new("sysctl")
.args(["-n", "hw.memsize"])
.output()
{
if let Ok(mem) = String::from_utf8(output.stdout)
.unwrap_or_default()
.trim()
.parse::<u64>()
{
total_mem = Some(mem);
}
}
if let Ok(output) = std::process::Command::new("sysctl")
.args(["-n", "hw.logicalcpu"])
.output()
{
if let Ok(cpus) = String::from_utf8(output.stdout)
.unwrap_or_default()
.trim()
.parse::<u32>()
{
total_cpus = Some(cpus);
}
}
self.nodes.push(NumaNode {
id: 0,
cpus: total_cpus.map(|n| (0..n).collect()),
memory_total_bytes: total_mem,
memory_free_bytes: None,
memory_used_bytes: None,
hugepages_total: None,
hugepages_free: None,
pci_devices: Vec::new(),
});
Ok(())
}
#[cfg_attr(not(target_os = "linux"), allow(dead_code))]
fn single_node_fallback(&mut self, cpus: Option<Vec<u32>>) {
self.nodes.push(NumaNode {
id: 0,
cpus,
memory_total_bytes: None,
memory_free_bytes: None,
memory_used_bytes: None,
hugepages_total: None,
hugepages_free: None,
pci_devices: Vec::new(),
});
self.distance_matrix = Some(NumaDistanceMatrix {
size: 1,
distances: vec![10],
});
}
}
impl Default for NumaMonitor {
fn default() -> Self {
Self::new().unwrap_or(Self {
nodes: Vec::new(),
distance_matrix: None,
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_numa_monitor_creation() {
match NumaMonitor::new() {
Ok(_monitor) => {}
Err(e) => {
let why = e.to_string();
assert!(
why.len() > 10,
"enumeration failed without saying why: {why:?}"
);
}
}
}
#[test]
fn test_numa_monitor_default() {
let monitor = NumaMonitor::default();
let summary = monitor.summary();
assert!(summary.node_count >= 1);
}
#[test]
fn test_distance_matrix_local() {
let matrix = NumaDistanceMatrix {
size: 2,
distances: vec![10, 21, 21, 10],
};
assert_eq!(matrix.distance(0, 0), Some(10));
assert_eq!(matrix.distance(0, 1), Some(21));
assert!(matrix.is_numa());
}
#[test]
fn test_uma_not_numa() {
let matrix = NumaDistanceMatrix {
size: 1,
distances: vec![10],
};
assert!(!matrix.is_numa());
}
#[test]
fn test_serialization() {
let node = NumaNode {
id: 0,
cpus: Some(vec![0, 1, 2, 3]),
memory_total_bytes: Some(16_000_000_000),
memory_free_bytes: Some(8_000_000_000),
memory_used_bytes: Some(8_000_000_000),
hugepages_total: Some(0),
hugepages_free: Some(0),
pci_devices: vec!["0000:00:02.0".into()],
};
let json = serde_json::to_string(&node).unwrap();
let _: NumaNode = serde_json::from_str(&json).unwrap();
}
#[cfg(target_os = "linux")]
#[test]
fn node_meminfo_reads_the_figure_not_the_node_id() {
let text = concat!(
"Node 0 MemTotal: 48111256 kB\n",
"Node 0 MemFree: 410056 kB\n",
"Node 0 MemUsed: 47701200 kB\n",
);
let (total, free, used) = NumaMonitor::parse_node_meminfo(text);
assert_eq!(total, Some(48_111_256 * 1024));
assert_eq!(free, Some(410_056 * 1024));
assert_eq!(used, Some((48_111_256 - 410_056) * 1024));
let node1 = "Node 1 MemTotal: 16000000 kB\n";
assert_eq!(
NumaMonitor::parse_node_meminfo(node1).0,
Some(16_000_000 * 1024)
);
}
#[cfg(target_os = "linux")]
#[test]
fn test_parse_cpu_list() {
assert_eq!(NumaMonitor::parse_cpu_list("0-3"), vec![0, 1, 2, 3]);
assert_eq!(NumaMonitor::parse_cpu_list("0,2,4"), vec![0, 2, 4]);
assert_eq!(NumaMonitor::parse_cpu_list("0-1,4-5"), vec![0, 1, 4, 5]);
}
}
#[cfg(test)]
mod absent_slit_tests {
use super::*;
fn node(id: u32) -> NumaNode {
NumaNode {
id,
cpus: None,
memory_total_bytes: None,
memory_free_bytes: None,
memory_used_bytes: None,
hugepages_total: None,
hugepages_free: None,
pci_devices: vec![],
}
}
fn node_with_cpus(id: u32, cpus: Vec<u32>) -> NumaNode {
NumaNode {
cpus: Some(cpus),
..node(id)
}
}
#[test]
fn an_unread_processor_list_is_not_an_empty_one() {
let monitor = NumaMonitor {
nodes: vec![node_with_cpus(0, vec![0, 1, 2, 3]), node(1)],
distance_matrix: None,
};
let summary = monitor.summary();
assert_eq!(summary.total_cpus, None);
assert_eq!(summary.cpu_imbalance_ratio, None);
assert_eq!(monitor.cpu_to_node(2), Some(0));
}
#[test]
fn read_processor_lists_total_and_balance() {
let monitor = NumaMonitor {
nodes: vec![
node_with_cpus(0, vec![0, 1]),
node_with_cpus(1, vec![2, 3, 4, 5]),
],
distance_matrix: None,
};
let summary = monitor.summary();
assert_eq!(summary.total_cpus, Some(6));
assert_eq!(summary.cpu_imbalance_ratio, Some(2.0));
}
#[test]
fn a_memory_only_node_with_no_processors_is_a_reading() {
let monitor = NumaMonitor {
nodes: vec![node_with_cpus(0, vec![0, 1]), node_with_cpus(1, vec![])],
distance_matrix: None,
};
assert_eq!(monitor.summary().total_cpus, Some(2));
}
#[test]
fn an_unreadable_distance_matrix_does_not_assert_local_memory() {
let monitor = NumaMonitor {
nodes: vec![node(0), node(1)],
distance_matrix: None,
};
assert_eq!(monitor.summary().max_distance, None);
}
#[test]
fn an_empty_distance_matrix_does_not_assert_local_memory() {
let monitor = NumaMonitor {
nodes: vec![node(0)],
distance_matrix: Some(NumaDistanceMatrix {
size: 0,
distances: vec![],
}),
};
assert_eq!(monitor.summary().max_distance, None);
}
#[test]
fn a_read_matrix_reports_its_maximum() {
let monitor = NumaMonitor {
nodes: vec![node(0), node(1)],
distance_matrix: Some(NumaDistanceMatrix {
size: 2,
distances: vec![10, 21, 21, 10],
}),
};
assert_eq!(monitor.summary().max_distance, Some(21));
}
}