use core::sync::atomic::AtomicU64;
use std::sync::Arc;
use crate::sched::injector::InjectorSteal as Steal;
use crate::numa_topology::numa_topology;
use crate::sched::arena_local::LocalArena;
use crate::sched::job::JobRef;
use crate::sched::numa_alloc::NUMA_NODE_LOCAL;
pub struct NumaArena {
nodes: Vec<Arc<LocalArena>>,
cpu_to_node: Vec<u32>,
cross_node_rotor: AtomicU64,
}
impl std::fmt::Debug for NumaArena {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("NumaArena")
.field("n_nodes", &self.nodes.len())
.field("workers_per_node",
&self.nodes.iter().map(|a| a.worker_count()).collect::<Vec<_>>())
.finish()
}
}
impl NumaArena {
pub fn debug_snapshot(&self) -> String {
self.nodes
.iter()
.enumerate()
.map(|(i, a)| format!("node {i}:\n{}", a.debug_snapshot()))
.collect()
}
}
impl NumaArena {
pub fn new(workers_per_node: Option<usize>) -> Arc<Self> {
let topo = numa_topology();
let n_nodes = topo.num_nodes.max(1) as usize;
let cpu_to_node = topo.node_of_cpu.clone();
let pinning_enabled = !pin_disabled_env_local();
let all_core_ids = if pinning_enabled {
core_affinity::get_core_ids()
} else {
None
};
let mut nodes: Vec<Arc<LocalArena>> = Vec::with_capacity(n_nodes);
for node_id in 0..n_nodes as u32 {
let cpus_in_node: Vec<u32> = topo.cpus_in_node(node_id);
let node_cpu_set: Option<Vec<core_affinity::CoreId>> = all_core_ids.as_ref().map(|ids| {
cpus_in_node
.iter()
.filter_map(|&cpu| ids.get(cpu as usize).copied())
.collect()
});
let cpu_info = crate::cpu_info::cpu_info();
let smt = (1usize << cpu_info.smt_threads_log2).max(1);
let phys_in_node = (cpus_in_node.len() / smt).max(1);
let primary_count = workers_per_node.unwrap_or(phys_in_node);
let smt_extension = if smt > 1 && primary_count < cpus_in_node.len() {
primary_count.saturating_mul(smt - 1)
} else {
0
};
nodes.push(LocalArena::with_smt_extension(
primary_count,
smt_extension,
node_cpu_set,
));
}
Arc::new(Self {
nodes,
cpu_to_node,
cross_node_rotor: AtomicU64::new(0),
})
}
pub fn node_count(&self) -> usize {
self.nodes.len()
}
pub fn migrate_all_workers_k_gating(&self, gating: super::k_gating::KGating) {
for arena in &self.nodes {
arena.migrate_all_workers_k_gating(gating);
}
}
pub fn global_burst_ratio(&self) -> f32 {
let mut bursts: u64 = 0;
let mut singles: u64 = 0;
for arena in &self.nodes {
for w in arena.worker_stats() {
use core::sync::atomic::Ordering;
bursts = bursts.saturating_add(w.burst_pushes.load(Ordering::Relaxed));
singles = singles.saturating_add(w.single_pushes.load(Ordering::Relaxed));
}
}
let total = bursts + singles;
if total == 0 {
0.5
} else {
(bursts as f32) / (total as f32)
}
}
pub fn total_workers(&self) -> usize {
self.nodes.iter().map(|a| a.worker_count()).sum()
}
pub fn iter_worker_stats(&self) -> impl Iterator<Item = &Arc<super::arena_local::WorkerStats>> {
self.nodes.iter().flat_map(|a| a.worker_stats().iter())
}
pub fn acquire_smt(&self) -> Vec<super::arena_local::SmtGuard> {
self.nodes.iter().map(|n| n.acquire_smt()).collect()
}
#[inline]
pub fn try_acquire_smt_single(&self) -> Option<super::arena_local::SmtGuard> {
if self.nodes.len() == 1 {
Some(self.nodes[0].acquire_smt())
} else {
None
}
}
#[inline]
pub fn is_single_numa(&self) -> bool {
self.nodes.len() == 1
}
#[inline]
pub fn single_node_arc(&self) -> Option<Arc<super::arena_local::LocalArena>> {
if self.nodes.len() == 1 {
Some(Arc::clone(&self.nodes[0]))
} else {
None
}
}
#[inline]
pub fn node_arc(&self, node_hint: Option<u32>) -> Arc<super::arena_local::LocalArena> {
Arc::clone(&self.nodes[self.resolve_node_idx(node_hint)])
}
fn resolve_node_idx(&self, node_hint: Option<u32>) -> usize {
match node_hint {
Some(n) if n != NUMA_NODE_LOCAL && (n as usize) < self.nodes.len() => n as usize,
_ => current_thread_node_idx(&self.cpu_to_node, self.nodes.len()),
}
}
pub(crate) unsafe fn submit(&self, job: JobRef, node_hint: Option<u32>) {
let idx = self.resolve_node_idx(node_hint);
unsafe { self.nodes[idx].submit(job) };
}
pub fn try_run_one(&self, rng_state: &mut u64) -> bool {
let local_idx = current_thread_node_idx(&self.cpu_to_node, self.nodes.len());
if self.nodes[local_idx].try_run_one(rng_state) {
return true;
}
if self.nodes.len() <= 1 {
return false;
}
let rotor = self.cross_node_rotor.fetch_add(1, core::sync::atomic::Ordering::Relaxed)
as usize
% (self.nodes.len() - 1);
for offset in 0..self.nodes.len() - 1 {
let candidate = (local_idx + 1 + (rotor + offset) % (self.nodes.len() - 1))
% self.nodes.len();
if candidate == local_idx {
continue;
}
if let Steal::Success(job) = self.nodes[candidate].injector_view().steal() {
unsafe { job.execute() };
return true;
}
}
false
}
}
fn pin_disabled_env_local() -> bool {
static CACHE: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*CACHE.get_or_init(|| {
match std::env::var("FLYNNEL_SCHED_PIN") {
Ok(v) => {
let v = v.to_ascii_lowercase();
if v == "on" || v == "1" || v == "true" {
false } else {
true
}
}
Err(_) => true, }
})
}
fn current_thread_node_idx(cpu_to_node: &[u32], n_nodes: usize) -> usize {
let cpu = current_thread_cpu();
let n = cpu_to_node.get(cpu).copied().unwrap_or(0) as usize;
n.min(n_nodes.saturating_sub(1))
}
#[cfg(target_os = "windows")]
fn current_thread_cpu() -> usize {
#[link(name = "kernel32")]
unsafe extern "system" {
fn GetCurrentProcessorNumber() -> u32;
}
unsafe { GetCurrentProcessorNumber() as usize }
}
#[cfg(target_os = "linux")]
fn current_thread_cpu() -> usize {
#[link(name = "c")]
unsafe extern "C" {
fn sched_getcpu() -> core::ffi::c_int;
}
let cpu = unsafe { sched_getcpu() };
if cpu < 0 { 0 } else { cpu as usize }
}
#[cfg(not(any(target_os = "windows", target_os = "linux")))]
fn current_thread_cpu() -> usize {
0
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicU32, Ordering};
use std::time::{Duration, Instant};
use crate::foundation::Variant;
use crate::sched::job::{NUMA_HINT_ANY, StackJob};
use crate::sched::latch::CoreLatch;
#[test]
fn numa_arena_collapses_to_single_arena_on_single_numa_host() {
let a = NumaArena::new(Some(4));
assert_eq!(a.node_count(), numa_topology().num_nodes as usize);
assert!(a.total_workers() >= 1);
}
#[test]
fn numa_arena_submit_runs_job() {
let arena = NumaArena::new(Some(2));
let counter = Arc::new(AtomicU32::new(0));
let c = Arc::clone(&counter);
let job = StackJob::new(
move |_stolen| { c.fetch_add(1, Ordering::SeqCst); },
CoreLatch::new(),
);
unsafe {
let r = job.as_job_ref(2, NUMA_HINT_ANY, Variant::Faithful);
arena.submit(r, None);
}
let deadline = Instant::now() + Duration::from_secs(5);
while !job.latch.is_set() {
if Instant::now() > deadline {
panic!("NumaArena job did not complete within 5s");
}
std::thread::yield_now();
}
assert_eq!(counter.load(Ordering::SeqCst), 1);
drop(arena);
}
#[test]
fn numa_arena_submit_many_completes() {
const N: u32 = 128;
let arena = NumaArena::new(Some(2));
let counter = Arc::new(AtomicU32::new(0));
let jobs: Vec<_> = (0..N)
.map(|_| {
let c = Arc::clone(&counter);
StackJob::new(
move |_stolen| { c.fetch_add(1, Ordering::SeqCst); },
CoreLatch::new(),
)
})
.collect();
for j in &jobs {
unsafe { arena.submit(j.as_job_ref(2, NUMA_HINT_ANY, Variant::Faithful), None) };
}
let deadline = Instant::now() + Duration::from_secs(10);
loop {
let pending = jobs.iter().filter(|j| !j.latch.is_set()).count();
if pending == 0 { break; }
if Instant::now() > deadline {
panic!("only {}/{N} jobs completed within 10s", N as usize - pending);
}
std::thread::yield_now();
}
assert_eq!(counter.load(Ordering::SeqCst), N);
drop(arena);
}
#[test]
fn numa_arena_node_hint_oob_clamps_to_local() {
let arena = NumaArena::new(Some(2));
let counter = Arc::new(AtomicU32::new(0));
let c = Arc::clone(&counter);
let job = StackJob::new(
move |_stolen| { c.fetch_add(1, Ordering::SeqCst); },
CoreLatch::new(),
);
unsafe {
arena.submit(
job.as_job_ref(2, NUMA_HINT_ANY, Variant::Faithful),
Some(99), );
}
let deadline = Instant::now() + Duration::from_secs(5);
while !job.latch.is_set() {
if Instant::now() > deadline {
panic!("OOB node-hint job did not complete");
}
std::thread::yield_now();
}
assert_eq!(counter.load(Ordering::SeqCst), 1);
drop(arena);
}
#[test]
fn numa_arena_try_run_one_returns_false_when_empty() {
let arena = NumaArena::new(Some(2));
let mut rng = 0x9E37_79B9_7F4A_7C15;
assert!(!arena.try_run_one(&mut rng),
"try_run_one must return false when no work pending");
drop(arena);
}
}