use std::cell::UnsafeCell;
use std::collections::BTreeSet;
use std::fmt::{self, Write as _};
use std::fs;
use std::io::ErrorKind;
use std::ops::Range;
use std::panic::{AssertUnwindSafe, catch_unwind, resume_unwind};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, PoisonError};
use std::thread::JoinHandle;
use std::time::Duration;
use thiserror::Error;
use tracing::{debug, info, warn};
const SYSFS_ROOT: &str = "/sys";
#[cfg(target_os = "linux")]
pub const MAX_CPUS: usize = rustix::thread::CpuSet::MAX_CPU;
#[cfg(not(target_os = "linux"))]
pub const MAX_CPUS: usize = 1024;
pub const MAX_SHARDS: usize = MAX_CPUS;
pub const SPIN_ROUNDS: u32 = 64;
const WAKE_ALL: u32 = i32::MAX.unsigned_abs();
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct CpuMask(BTreeSet<usize>);
impl CpuMask {
#[must_use]
pub fn new() -> Self {
Self(BTreeSet::new())
}
#[must_use]
pub fn up_to(n: usize) -> Self {
Self((0..n.min(MAX_CPUS)).collect())
}
pub fn insert(&mut self, cpu: usize) -> bool {
if cpu >= MAX_CPUS {
return false;
}
self.0.insert(cpu)
}
#[must_use]
pub fn contains(&self, cpu: usize) -> bool {
self.0.contains(&cpu)
}
#[must_use]
pub fn len(&self) -> usize {
self.0.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
pub fn iter(&self) -> impl Iterator<Item = usize> + '_ {
self.0.iter().copied()
}
#[must_use]
pub fn intersect(&self, other: &Self) -> Self {
Self(self.0.intersection(&other.0).copied().collect())
}
#[must_use]
pub fn to_vec(&self) -> Vec<usize> {
self.0.iter().copied().collect()
}
#[must_use]
pub fn current() -> Option<Self> {
current_affinity()
}
}
impl FromIterator<usize> for CpuMask {
fn from_iter<I: IntoIterator<Item = usize>>(iter: I) -> Self {
let mut mask = Self::new();
for cpu in iter {
mask.insert(cpu);
}
mask
}
}
impl fmt::Display for CpuMask {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&format_cpu_list(&self.to_vec()))
}
}
#[cfg(target_os = "linux")]
fn current_affinity() -> Option<CpuMask> {
match rustix::thread::sched_getaffinity(None) {
Ok(set) => {
let mut mask = CpuMask::new();
for cpu in 0..MAX_CPUS {
if set.is_set(cpu) {
mask.insert(cpu);
}
}
if mask.is_empty() { None } else { Some(mask) }
}
Err(err) => {
debug!(error = %err, "sched_getaffinity failed; assuming all cpus");
None
}
}
}
#[cfg(not(target_os = "linux"))]
fn current_affinity() -> Option<CpuMask> {
None
}
fn affinity_or_default() -> CpuMask {
current_affinity().unwrap_or_else(|| CpuMask::up_to(available_parallelism()))
}
fn available_parallelism() -> usize {
std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get)
}
#[derive(Debug, Error)]
pub enum PinError {
#[error("cpu {cpu} is outside this process's affinity mask")]
NotAllowed {
cpu: usize,
},
#[error("cpu {cpu} is beyond the {max}-cpu affinity mask limit")]
OutOfRange {
cpu: usize,
max: usize,
},
#[error("refusing to pin to an empty cpu mask")]
EmptyMask,
#[error("sched_setaffinity failed: {0}")]
SetAffinity(#[source] std::io::Error),
#[error("cpu pinning is not supported on this platform")]
Unsupported,
}
pub fn pin_current_thread(cpu: usize) -> Result<(), PinError> {
if cpu >= MAX_CPUS {
return Err(PinError::OutOfRange { cpu, max: MAX_CPUS });
}
if let Some(allowed) = current_affinity()
&& !allowed.contains(cpu)
{
return Err(PinError::NotAllowed { cpu });
}
let mut mask = CpuMask::new();
mask.insert(cpu);
set_affinity(&mask)
}
pub fn pin_current_thread_to(mask: &CpuMask) -> Result<(), PinError> {
if mask.is_empty() {
return Err(PinError::EmptyMask);
}
set_affinity(mask)
}
#[cfg(target_os = "linux")]
fn set_affinity(mask: &CpuMask) -> Result<(), PinError> {
let mut set = rustix::thread::CpuSet::new();
for cpu in mask.iter() {
if cpu >= MAX_CPUS {
return Err(PinError::OutOfRange { cpu, max: MAX_CPUS });
}
set.set(cpu);
}
rustix::thread::sched_setaffinity(None, &set)
.map_err(|err| PinError::SetAffinity(std::io::Error::from(err)))
}
#[cfg(not(target_os = "linux"))]
fn set_affinity(_mask: &CpuMask) -> Result<(), PinError> {
Err(PinError::Unsupported)
}
fn place_or_warn(mask: &CpuMask, what: &'static str) -> bool {
let mut cpus = mask.iter();
if let (Some(cpu), None) = (cpus.next(), cpus.next()) {
return pin_or_warn(cpu, what);
}
match pin_current_thread_to(mask) {
Ok(()) => {
debug!(cpus = %mask, thread = what, "placed thread");
true
}
Err(err) => {
warn!(cpus = %mask, thread = what, error = %err, "cpu placement failed; running unplaced");
false
}
}
}
fn pin_or_warn(cpu: usize, what: &'static str) -> bool {
match pin_current_thread(cpu) {
Ok(()) => {
debug!(cpu, thread = what, "pinned thread");
true
}
Err(err) => {
warn!(cpu, thread = what, error = %err, "cpu pin failed; running unpinned");
false
}
}
}
#[derive(Debug, Error)]
pub enum TopologyError {
#[error("{}: {source}", path.display())]
Io {
path: PathBuf,
#[source]
source: std::io::Error,
},
#[error("{}: malformed cpu list {value:?} ({reason})", path.display())]
Malformed {
path: PathBuf,
value: String,
reason: &'static str,
},
#[error("the process affinity mask is empty")]
NoCpus,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Topology {
allowed: CpuMask,
performance: Vec<usize>,
efficiency: Vec<usize>,
l3_efficiency: Vec<usize>,
compute: Vec<usize>,
reactor: Option<usize>,
hybrid: bool,
pins: bool,
shards: usize,
}
impl Topology {
#[must_use]
pub fn detect() -> Self {
Self::detect_at(Path::new(SYSFS_ROOT), affinity_or_default())
}
#[must_use]
pub fn detect_at(sysfs: &Path, allowed: CpuMask) -> Self {
match Self::parse_at(sysfs, allowed.clone()) {
Ok(topo) => {
if topo.pins {
info!(
compute = %format_cpu_list(&topo.compute),
reactor = ?topo.reactor,
"hybrid cpu topology detected; pinning compute threads"
);
} else {
info!(
shards = topo.shards,
"no pinnable hybrid topology; running unpinned"
);
}
topo
}
Err(err) => {
debug!(error = %err, "cpu topology detection failed; running unpinned");
Self::unpinned(allowed)
}
}
}
#[must_use]
pub fn unpinned(allowed: CpuMask) -> Self {
let shards = fallback_shards(&allowed);
Self {
allowed,
performance: Vec::new(),
efficiency: Vec::new(),
l3_efficiency: Vec::new(),
compute: Vec::new(),
reactor: None,
hybrid: false,
pins: false,
shards,
}
}
pub fn parse_at(sysfs: &Path, allowed: CpuMask) -> Result<Self, TopologyError> {
if allowed.is_empty() {
return Err(TopologyError::NoCpus);
}
let perf = read_cpu_list(&sysfs.join("devices/cpu_core/cpus"))?;
let eff = read_cpu_list(&sysfs.join("devices/cpu_atom/cpus"))?;
let hybrid = matches!((&perf, &eff), (Some(p), Some(e)) if !p.is_empty() && !e.is_empty());
let performance: Vec<usize> = perf
.unwrap_or_default()
.into_iter()
.filter(|c| allowed.contains(*c))
.collect();
let efficiency: Vec<usize> = eff
.unwrap_or_default()
.into_iter()
.filter(|c| allowed.contains(*c))
.collect();
let candidates: Vec<usize> = if hybrid {
performance.clone()
} else {
let online = read_cpu_list(&sysfs.join("devices/system/cpu/online"))?;
match online {
Some(list) => list.into_iter().filter(|c| allowed.contains(*c)).collect(),
None => allowed.to_vec(),
}
};
let compute = smt_primaries(sysfs, &candidates).unwrap_or_default();
let l3_efficiency: Vec<usize> = efficiency
.iter()
.copied()
.filter(|&cpu| cpu_has_l3(sysfs, cpu))
.collect();
let reactor = if hybrid {
l3_efficiency.first().copied()
} else {
None
};
let pins = hybrid && compute.len() >= 2;
let shards = if pins {
compute.len()
} else {
fallback_shards(&allowed)
};
Ok(Self {
allowed,
performance,
efficiency,
l3_efficiency,
compute: if pins { compute } else { Vec::new() },
reactor,
hybrid,
pins,
shards,
})
}
#[must_use]
pub fn is_hybrid(&self) -> bool {
self.hybrid
}
#[must_use]
pub fn pins(&self) -> bool {
self.pins
}
#[must_use]
pub fn allowed(&self) -> &CpuMask {
&self.allowed
}
#[must_use]
pub fn performance_cpus(&self) -> &[usize] {
&self.performance
}
#[must_use]
pub fn efficiency_cpus(&self) -> &[usize] {
&self.efficiency
}
#[must_use]
pub fn l3_efficiency_cpus(&self) -> &[usize] {
&self.l3_efficiency
}
#[must_use]
pub fn shared_cpus(&self) -> CpuMask {
self.performance
.iter()
.chain(&self.l3_efficiency)
.copied()
.collect()
}
#[must_use]
pub fn compute_cpus(&self) -> &[usize] {
&self.compute
}
#[must_use]
pub fn reactor_cpu(&self) -> Option<usize> {
self.reactor
}
#[must_use]
pub fn shard_count(&self) -> usize {
self.shards.max(1)
}
}
impl fmt::Display for Topology {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"{} cpus allowed {}",
if self.hybrid { "hybrid" } else { "uniform" },
format_cpu_list(&self.allowed.to_vec())
)?;
if self.hybrid {
write!(
f,
"; P {}; E {} (L3 {})",
format_cpu_list(&self.performance),
format_cpu_list(&self.efficiency),
format_cpu_list(&self.l3_efficiency)
)?;
}
write!(f, "; compute {}", format_cpu_list(&self.compute))?;
match self.reactor {
Some(cpu) => write!(f, "; reactor {cpu}")?,
None => write!(f, "; reactor unpinned")?,
}
write!(
f,
"; {} shards, pinning {}",
self.shard_count(),
if self.pins { "on" } else { "off" }
)
}
}
fn fallback_shards(allowed: &CpuMask) -> usize {
allowed.len().min(available_parallelism()).max(1)
}
fn read_cpu_list(path: &Path) -> Result<Option<Vec<usize>>, TopologyError> {
let text = match fs::read_to_string(path) {
Ok(text) => text,
Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None),
Err(source) => {
return Err(TopologyError::Io {
path: path.to_path_buf(),
source,
});
}
};
parse_cpu_list(text.trim())
.map(Some)
.map_err(|reason| TopologyError::Malformed {
path: path.to_path_buf(),
value: text.trim().to_owned(),
reason,
})
}
fn parse_cpu_list(text: &str) -> Result<Vec<usize>, &'static str> {
let mut cpus = BTreeSet::new();
let text = text.trim();
if text.is_empty() {
return Ok(Vec::new());
}
for part in text.split(',') {
let part = part.trim();
if part.is_empty() {
return Err("empty range element");
}
let (lo, hi) = match part.split_once('-') {
Some((lo, hi)) => (lo.trim(), hi.trim()),
None => (part, part),
};
let lo: usize = lo.parse().map_err(|_| "not a cpu id")?;
let hi: usize = hi.parse().map_err(|_| "not a cpu id")?;
if hi < lo {
return Err("descending range");
}
if hi >= MAX_CPUS {
return Err("cpu id beyond the affinity mask limit");
}
for cpu in lo..=hi {
cpus.insert(cpu);
}
}
Ok(cpus.into_iter().collect())
}
fn format_cpu_list(cpus: &[usize]) -> String {
if cpus.is_empty() {
return "-".to_owned();
}
let mut out = String::new();
let mut i = 0;
while i < cpus.len() {
let start = cpus[i];
let mut end = start;
let mut j = i + 1;
while j < cpus.len() && cpus[j] == end + 1 {
end = cpus[j];
j += 1;
}
if !out.is_empty() {
out.push(',');
}
let _ = if end == start {
write!(out, "{start}")
} else {
write!(out, "{start}-{end}")
};
i = j;
}
out
}
fn smt_primaries(sysfs: &Path, candidates: &[usize]) -> Option<Vec<usize>> {
let mut claimed: BTreeSet<usize> = BTreeSet::new();
let mut primaries = Vec::new();
for &cpu in candidates {
if claimed.contains(&cpu) {
continue;
}
let siblings = read_siblings(sysfs, cpu)?;
primaries.push(cpu);
claimed.insert(cpu);
claimed.extend(siblings);
}
Some(primaries)
}
fn read_siblings(sysfs: &Path, cpu: usize) -> Option<Vec<usize>> {
let base = sysfs.join(format!("devices/system/cpu/cpu{cpu}/topology"));
for name in ["thread_siblings_list", "core_cpus_list"] {
match read_cpu_list(&base.join(name)) {
Ok(Some(list)) if !list.is_empty() => return Some(list),
Ok(_) => {}
Err(err) => warn!(cpu, file = name, error = %err, "unusable smt sibling list"),
}
}
warn!(
cpu,
topology = %base.display(),
"sysfs reports no usable smt sibling list; refusing to pin, because \
treating an unknown sibling list as 'no sibling' would put two \
compute threads on one physical core"
);
None
}
fn cpu_has_l3(sysfs: &Path, cpu: usize) -> bool {
let dir = sysfs.join(format!("devices/system/cpu/cpu{cpu}/cache"));
let entries = match fs::read_dir(&dir) {
Ok(entries) => entries,
Err(err) => {
if err.kind() != ErrorKind::NotFound {
debug!(cpu, error = %err, "cannot list cache nodes; assuming no L3");
}
return false;
}
};
for entry in entries.flatten() {
if !entry.file_name().as_encoded_bytes().starts_with(b"index") {
continue;
}
if let Ok(text) = fs::read_to_string(entry.path().join("level"))
&& text.trim() == "3"
{
return true;
}
}
false
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Shard {
pub index: usize,
pub count: usize,
pub rows: Range<usize>,
}
impl Shard {
#[must_use]
pub fn len(&self) -> usize {
self.rows.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.rows.is_empty()
}
}
#[must_use]
pub fn shard_range(rows: usize, shards: usize, index: usize) -> Range<usize> {
if shards == 0 || index >= shards {
return 0..0;
}
let base = rows / shards;
let rem = rows % shards;
let start = index * base + index.min(rem);
let len = base + usize::from(index < rem);
start..start + len
}
pub const CHUNKS_PER_SHARD: usize = 4;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Chunk {
pub index: usize,
pub count: usize,
pub rows: Range<usize>,
pub on_caller: bool,
}
#[must_use]
pub fn chunk_count(rows: usize, shards: usize, per_shard: usize) -> usize {
shards.saturating_mul(per_shard).clamp(1, rows.max(1))
}
const BARRIER_POLL: Duration = Duration::from_millis(1);
const JOB_POLL_MIN: Duration = Duration::from_millis(1);
const JOB_POLL_MAX: Duration = Duration::from_millis(256);
#[cfg(target_os = "linux")]
mod park {
use rustix::thread::futex;
use std::sync::atomic::AtomicU32;
use std::time::Duration;
pub(super) fn wait(word: &AtomicU32, expected: u32, timeout: Duration) {
let timeout = futex::Timespec {
tv_sec: futex::Secs::try_from(timeout.as_secs()).unwrap_or(futex::Secs::MAX),
tv_nsec: timeout.subsec_nanos() as futex::Nsecs,
};
let _ = futex::wait(word, futex::Flags::PRIVATE, expected, Some(&timeout));
}
pub(super) fn wake(word: &AtomicU32, count: u32) {
let _ = futex::wake(word, futex::Flags::PRIVATE, count);
}
}
#[cfg(not(target_os = "linux"))]
mod park {
use std::sync::atomic::{AtomicU32, Ordering};
use std::time::{Duration, Instant};
pub(super) fn wait(word: &AtomicU32, expected: u32, timeout: Duration) {
let deadline = Instant::now() + timeout;
while word.load(Ordering::Acquire) == expected && Instant::now() < deadline {
std::thread::yield_now();
}
}
pub(super) fn wake(_word: &AtomicU32, _count: u32) {}
}
type PanicPayload = Box<dyn std::any::Any + Send + 'static>;
#[derive(Clone, Copy)]
struct JobRef {
data: *const (),
work: Work,
rows: usize,
}
#[derive(Clone, Copy)]
enum Work {
Shards {
call: unsafe fn(*const (), Shard),
shards: usize,
},
Chunks {
call: unsafe fn(*const (), Chunk),
count: usize,
},
}
unsafe fn call_job<F: Fn(Shard) + Sync>(data: *const (), shard: Shard) {
let f = unsafe { &*data.cast::<F>() };
f(shard);
}
unsafe fn call_chunk<F: Fn(Chunk) + Sync>(data: *const (), chunk: Chunk) {
let f = unsafe { &*data.cast::<F>() };
f(chunk);
}
struct Shared {
seq: AtomicU32,
pending: AtomicU32,
workers_parked: AtomicU32,
lead_parked: AtomicU32,
quit: AtomicU32,
next_chunk: AtomicUsize,
job: UnsafeCell<Option<JobRef>>,
panic: Mutex<Option<PanicPayload>>,
#[cfg(test)]
probe: tests::BarrierProbe,
}
unsafe impl Send for Shared {}
unsafe impl Sync for Shared {}
impl Shared {
fn new() -> Self {
Self {
seq: AtomicU32::new(0),
pending: AtomicU32::new(0),
workers_parked: AtomicU32::new(0),
lead_parked: AtomicU32::new(0),
quit: AtomicU32::new(0),
next_chunk: AtomicUsize::new(0),
job: UnsafeCell::new(None),
panic: Mutex::new(None),
#[cfg(test)]
probe: tests::BarrierProbe::new(),
}
}
fn record_panic(&self, payload: PanicPayload) {
let mut slot = self.panic.lock().unwrap_or_else(PoisonError::into_inner);
if slot.is_none() {
*slot = Some(payload);
}
}
fn take_panic(&self) -> Option<PanicPayload> {
self.panic
.lock()
.unwrap_or_else(PoisonError::into_inner)
.take()
}
fn claim(&self, count: usize) -> Option<usize> {
let index = self.next_chunk.fetch_add(1, Ordering::Relaxed);
(index < count).then_some(index)
}
fn publish(&self, job: JobRef, workers: u32) {
unsafe {
*self.job.get() = Some(job);
}
self.next_chunk.store(0, Ordering::Relaxed);
self.pending.store(workers, Ordering::Relaxed);
let next = self.seq.load(Ordering::Relaxed).wrapping_add(1);
self.seq.store(next, Ordering::SeqCst);
if self.workers_parked.load(Ordering::SeqCst) > 0 {
park::wake(&self.seq, workers.min(WAKE_ALL));
}
}
fn finish_shard(&self) {
if self.pending.fetch_sub(1, Ordering::SeqCst) == 1 {
#[cfg(test)]
let stalled = self.probe.enter_wake_window();
let parked = self.lead_parked.load(Ordering::SeqCst) != 0;
if parked {
park::wake(&self.pending, 1);
}
#[cfg(test)]
if stalled {
self.probe.leave_wake_window(parked);
}
}
}
fn await_barrier(&self) {
for _ in 0..SPIN_ROUNDS {
if self.pending.load(Ordering::Acquire) == 0 {
return;
}
std::hint::spin_loop();
}
self.lead_parked.store(1, Ordering::SeqCst);
loop {
let outstanding = self.pending.load(Ordering::SeqCst);
if outstanding == 0 {
break;
}
#[cfg(test)]
self.probe.enter_park_window(&self.lead_parked);
park::wait(&self.pending, outstanding, BARRIER_POLL);
}
self.lead_parked.store(0, Ordering::SeqCst);
}
fn await_seq(&self, last: u32) -> u32 {
for _ in 0..SPIN_ROUNDS {
let seq = self.seq.load(Ordering::Acquire);
if seq != last {
return seq;
}
std::hint::spin_loop();
}
let mut poll = JOB_POLL_MIN;
loop {
self.workers_parked.fetch_add(1, Ordering::SeqCst);
let seq = self.seq.load(Ordering::SeqCst);
if seq != last {
self.workers_parked.fetch_sub(1, Ordering::SeqCst);
return seq;
}
park::wait(&self.seq, last, poll);
poll = (poll * 2).min(JOB_POLL_MAX);
self.workers_parked.fetch_sub(1, Ordering::SeqCst);
let seq = self.seq.load(Ordering::Acquire);
if seq != last {
return seq;
}
}
}
}
struct BarrierGuard<'a> {
shared: &'a Shared,
}
impl Drop for BarrierGuard<'_> {
fn drop(&mut self) {
self.shared.await_barrier();
unsafe {
*self.shared.job.get() = None;
}
if std::thread::panicking() {
if self.shared.take_panic().is_some() {
warn!("discarding worker panic while the submitting thread unwinds");
}
}
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum Placement {
PerCore,
#[default]
Shared,
Off,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct PoolConfig {
pub shards: Option<usize>,
pub placement: Placement,
pub pin_caller: bool,
pub inline_caller: bool,
pub chunks_per_shard: usize,
}
impl Default for PoolConfig {
fn default() -> Self {
Self {
shards: None,
placement: Placement::Shared,
pin_caller: true,
inline_caller: true,
chunks_per_shard: CHUNKS_PER_SHARD,
}
}
}
pub struct ComputePool {
shared: Arc<Shared>,
workers: Vec<JoinHandle<()>>,
topology: Topology,
shards: usize,
inline_caller: bool,
chunks_per_shard: usize,
placement: Placement,
lead_cpu: Option<usize>,
caller: Option<CpuMask>,
}
impl ComputePool {
#[must_use]
pub fn new() -> Self {
Self::with_config(PoolConfig::default())
}
#[must_use]
pub fn with_config(config: PoolConfig) -> Self {
let topology = Topology::detect();
Self::with_topology(topology, config)
}
#[must_use]
pub fn with_topology(topology: Topology, config: PoolConfig) -> Self {
let shards = config
.shards
.unwrap_or_else(|| topology.shard_count())
.clamp(1, MAX_SHARDS);
let inline_caller = config.inline_caller;
let placement = if topology.pins() {
config.placement
} else {
Placement::Off
};
let compute = topology.compute_cpus().to_vec();
let shared_cpus = topology.shared_cpus();
let offset = usize::from(inline_caller);
let worker_count = shards.saturating_sub(offset);
let lead_cpu = if placement == Placement::PerCore && inline_caller {
compute.first().copied()
} else {
None
};
let caller = match placement {
Placement::PerCore => lead_cpu.map(|cpu| std::iter::once(cpu).collect()),
Placement::Shared => Some(shared_cpus.clone()),
Placement::Off => None,
};
let shared = Arc::new(Shared::new());
let mut workers = Vec::with_capacity(worker_count);
for i in 0..worker_count {
let shard_index = i + offset;
let place: Option<CpuMask> = match placement {
Placement::PerCore => compute
.get(shard_index)
.map(|&cpu| std::iter::once(cpu).collect()),
Placement::Shared => Some(shared_cpus.clone()),
Placement::Off => None,
};
let shared = Arc::clone(&shared);
let spawned = std::thread::Builder::new()
.name(format!("rvmp-compute-{shard_index}"))
.spawn(move || worker_loop(&shared, shard_index, place));
match spawned {
Ok(handle) => workers.push(handle),
Err(err) => {
warn!(
error = %err,
wanted = worker_count,
got = workers.len(),
"could not spawn compute worker; shrinking the pool"
);
break;
}
}
}
let shards = workers.len() + offset;
let pool = Self {
shared,
workers,
topology,
shards: shards.max(1),
inline_caller,
chunks_per_shard: config.chunks_per_shard.max(1),
placement,
lead_cpu,
caller,
};
if config.pin_caller {
pool.pin_calling_thread();
}
debug!(
shards = pool.shards,
workers = pool.workers.len(),
inline_caller,
?placement,
"compute pool ready"
);
pool
}
#[must_use]
pub fn shards(&self) -> usize {
self.shards
}
#[must_use]
pub fn topology(&self) -> &Topology {
&self.topology
}
#[must_use]
pub fn placement(&self) -> Placement {
self.placement
}
#[must_use]
pub fn lead_cpu(&self) -> Option<usize> {
self.lead_cpu
}
pub fn pin_calling_thread(&self) -> bool {
match &self.caller {
Some(mask) => place_or_warn(mask, "compute-lead"),
None => false,
}
}
pub fn run<F>(&mut self, rows: usize, f: F)
where
F: Fn(Shard) + Sync,
{
if rows == 0 {
return;
}
let shards = self.shards;
let inline_caller = self.inline_caller;
let workers = self.workers.len() as u32;
if workers == 0 || rows < shards {
f(Shard {
index: 0,
count: 1,
rows: 0..rows,
});
return;
}
let job = JobRef {
data: std::ptr::from_ref(&f).cast::<()>(),
work: Work::Shards {
call: call_job::<F>,
shards,
},
rows,
};
let shared: &Shared = &self.shared;
let guard = BarrierGuard { shared };
shared.publish(job, workers);
if inline_caller {
f(Shard {
index: 0,
count: shards,
rows: shard_range(rows, shards, 0),
});
}
drop(guard);
if let Some(payload) = shared.take_panic() {
resume_unwind(payload);
}
}
pub fn scatter<T, F>(&mut self, out: &mut [T], f: F)
where
T: Send,
F: Fn(Shard, &mut [T]) + Sync,
{
let rows = out.len();
let base = SendPtr(out.as_mut_ptr());
self.run(rows, |shard| {
let start = shard.rows.start;
let len = shard.rows.len();
let chunk = unsafe { std::slice::from_raw_parts_mut(base.get().add(start), len) };
f(shard, chunk);
});
}
#[must_use]
pub fn chunks(&self, rows: usize) -> usize {
if self.workers.is_empty() {
return 1;
}
chunk_count(rows, self.shards, self.chunks_per_shard)
}
pub fn run_chunks<F>(&mut self, rows: usize, f: F)
where
F: Fn(Chunk) + Sync,
{
if rows == 0 {
return;
}
let count = self.chunks(rows);
let workers = self.workers.len() as u32;
if count == 1 {
f(Chunk {
index: 0,
count: 1,
rows: 0..rows,
on_caller: true,
});
return;
}
let job = JobRef {
data: std::ptr::from_ref(&f).cast::<()>(),
work: Work::Chunks {
call: call_chunk::<F>,
count,
},
rows,
};
let shared: &Shared = &self.shared;
let guard = BarrierGuard { shared };
shared.publish(job, workers);
while let Some(index) = shared.claim(count) {
f(Chunk {
index,
count,
rows: shard_range(rows, count, index),
on_caller: true,
});
}
drop(guard);
if let Some(payload) = shared.take_panic() {
resume_unwind(payload);
}
}
pub fn scatter_chunks<T, F>(&mut self, out: &mut [T], f: F)
where
T: Send,
F: Fn(Chunk, &mut [T]) + Sync,
{
let rows = out.len();
let base = SendPtr(out.as_mut_ptr());
self.run_chunks(rows, |chunk| {
let start = chunk.rows.start;
let len = chunk.rows.len();
let piece = unsafe { std::slice::from_raw_parts_mut(base.get().add(start), len) };
f(chunk, piece);
});
}
}
impl Default for ComputePool {
fn default() -> Self {
Self::new()
}
}
impl fmt::Debug for ComputePool {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ComputePool")
.field("shards", &self.shards)
.field("workers", &self.workers.len())
.field("inline_caller", &self.inline_caller)
.field("chunks_per_shard", &self.chunks_per_shard)
.field("placement", &self.placement)
.field("lead_cpu", &self.lead_cpu)
.finish()
}
}
impl Drop for ComputePool {
fn drop(&mut self) {
self.shared.quit.store(1, Ordering::Release);
let next = self.shared.seq.load(Ordering::Relaxed).wrapping_add(1);
self.shared.seq.store(next, Ordering::SeqCst);
park::wake(&self.shared.seq, WAKE_ALL);
for handle in self.workers.drain(..) {
if handle.join().is_err() {
warn!("compute worker thread terminated abnormally");
}
}
}
}
struct SendPtr<T>(*mut T);
impl<T> SendPtr<T> {
fn get(&self) -> *mut T {
self.0
}
}
unsafe impl<T: Send> Send for SendPtr<T> {}
unsafe impl<T: Send> Sync for SendPtr<T> {}
struct ShardGuard<'a> {
shared: &'a Shared,
}
impl Drop for ShardGuard<'_> {
fn drop(&mut self) {
self.shared.finish_shard();
}
}
fn worker_loop(shared: &Shared, shard_index: usize, place: Option<CpuMask>) {
if let Some(mask) = place {
place_or_warn(&mask, "compute-worker");
}
let mut last = 0u32;
loop {
last = shared.await_seq(last);
if shared.quit.load(Ordering::Acquire) != 0 {
break;
}
let job = unsafe { *shared.job.get() };
let _retire = ShardGuard { shared };
match job.map(|job| (job.data, job.rows, job.work)) {
None => {}
Some((data, rows, Work::Shards { call, shards })) => {
let rows = shard_range(rows, shards, shard_index);
if !rows.is_empty() {
let shard = Shard {
index: shard_index,
count: shards,
rows,
};
contain(shared, || unsafe { call(data, shard) });
}
}
Some((data, rows, Work::Chunks { call, count })) => {
while let Some(index) = shared.claim(count) {
let chunk = Chunk {
index,
count,
rows: shard_range(rows, count, index),
on_caller: false,
};
contain(shared, || unsafe { call(data, chunk) });
}
}
}
}
}
fn contain(shared: &Shared, piece: impl FnOnce()) {
if let Err(payload) = catch_unwind(AssertUnwindSafe(piece)) {
if let Err(second) = catch_unwind(AssertUnwindSafe(|| shared.record_panic(payload))) {
std::mem::forget(second);
warn!("panic while recording a worker panic; payload leaked");
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicU32, AtomicUsize};
use std::time::{Instant, SystemTime, UNIX_EPOCH};
const PROBE_DEADLINE: Duration = Duration::from_secs(10);
#[derive(Debug, Default)]
pub(super) struct BarrierProbe {
stall_worker: AtomicU32,
worker_waiting: AtomicU32,
resume_worker: AtomicU32,
worker_left: AtomicU32,
worker_saw_flag: AtomicU32,
stall_lead: AtomicU32,
flag_after_straggler: AtomicU32,
flag_recorded: AtomicU32,
}
impl BarrierProbe {
pub(super) fn new() -> Self {
Self::default()
}
fn arm_worker(&self) {
self.stall_worker.store(1, Ordering::SeqCst);
}
fn arm_lead(&self) {
self.stall_lead.store(1, Ordering::SeqCst);
}
fn straggler_went_through_window(&self) -> bool {
self.worker_left.load(Ordering::SeqCst) != 0
}
fn worker_saw_flag(&self) -> bool {
self.worker_saw_flag.load(Ordering::SeqCst) != 0
}
fn flag_survived_straggler(&self) -> (bool, bool) {
(
self.flag_recorded.load(Ordering::SeqCst) != 0,
self.flag_after_straggler.load(Ordering::SeqCst) != 0,
)
}
pub(super) fn enter_wake_window(&self) -> bool {
if self.stall_worker.swap(0, Ordering::SeqCst) == 0 {
return false;
}
self.worker_waiting.store(1, Ordering::SeqCst);
wait_for(&self.resume_worker);
true
}
pub(super) fn leave_wake_window(&self, saw_flag: bool) {
self.worker_saw_flag
.store(u32::from(saw_flag), Ordering::SeqCst);
self.worker_left.store(1, Ordering::SeqCst);
}
pub(super) fn enter_park_window(&self, lead_parked: &AtomicU32) {
if self.stall_lead.load(Ordering::SeqCst) == 0
|| self.worker_waiting.load(Ordering::SeqCst) == 0
{
return;
}
self.resume_worker.store(1, Ordering::SeqCst);
wait_for(&self.worker_left);
self.flag_after_straggler
.store(lead_parked.load(Ordering::SeqCst), Ordering::SeqCst);
self.flag_recorded.store(1, Ordering::SeqCst);
self.stall_lead.store(0, Ordering::SeqCst);
}
}
fn wait_for(flag: &AtomicU32) {
let deadline = Instant::now() + PROBE_DEADLINE;
while flag.load(Ordering::SeqCst) == 0 && Instant::now() < deadline {
std::thread::sleep(Duration::from_micros(200));
}
}
struct TempDir(PathBuf);
impl TempDir {
fn new(tag: &str) -> Self {
static COUNTER: AtomicUsize = AtomicUsize::new(0);
let n = COUNTER.fetch_add(1, Ordering::Relaxed);
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_nanos();
let path = std::env::temp_dir().join(format!(
"ramvamp-threads-{tag}-{}-{n}-{nanos}",
std::process::id()
));
fs::create_dir_all(&path).expect("create test temp dir");
Self(path)
}
fn path(&self) -> &Path {
&self.0
}
}
impl Drop for TempDir {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.0);
}
}
fn write_file(root: &Path, rel: &str, contents: &str) {
let path = root.join(rel);
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).expect("create fixture dir");
}
fs::write(&path, contents).expect("write fixture file");
}
fn write_cpu(root: &Path, cpu: usize, siblings: &str, cache_levels: &[u32]) {
write_file(
root,
&format!("devices/system/cpu/cpu{cpu}/topology/thread_siblings_list"),
siblings,
);
write_file(
root,
&format!("devices/system/cpu/cpu{cpu}/cache/uevent"),
"MAJOR=0\n",
);
for (i, level) in cache_levels.iter().enumerate() {
write_file(
root,
&format!("devices/system/cpu/cpu{cpu}/cache/index{i}/level"),
&format!("{level}\n"),
);
}
}
fn reference_sysfs() -> TempDir {
let tmp = TempDir::new("185h");
let root = tmp.path();
write_file(root, "devices/cpu_core/cpus", "0-11\n");
write_file(root, "devices/cpu_atom/cpus", "12-21\n");
write_file(root, "devices/system/cpu/online", "0-21\n");
const PAIRS: [(usize, usize); 6] = [(0, 5), (1, 2), (3, 4), (6, 7), (8, 9), (10, 11)];
for (a, b) in PAIRS {
let list = format!("{a},{b}\n");
write_cpu(root, a, &list, &[1, 1, 2, 3]);
write_cpu(root, b, &list, &[1, 1, 2, 3]);
}
for cpu in 12..=19 {
write_cpu(root, cpu, &format!("{cpu}\n"), &[1, 1, 2, 3]);
}
for cpu in 20..=21 {
write_cpu(root, cpu, &format!("{cpu}\n"), &[1, 1, 2]);
}
tmp
}
fn uniform_sysfs() -> TempDir {
let tmp = TempDir::new("uniform");
let root = tmp.path();
write_file(root, "devices/system/cpu/online", "0-7\n");
for core in 0..4 {
let (a, b) = (core, core + 4);
let list = format!("{a},{b}\n");
write_cpu(root, a, &list, &[1, 1, 2, 3]);
write_cpu(root, b, &list, &[1, 1, 2, 3]);
}
tmp
}
#[test]
fn cpu_lists_parse_every_sysfs_spelling() {
assert_eq!(parse_cpu_list("0-11"), Ok((0..=11).collect::<Vec<_>>()));
assert_eq!(parse_cpu_list("0,5"), Ok(vec![0, 5]));
assert_eq!(
parse_cpu_list("0-3,8,10-11"),
Ok(vec![0, 1, 2, 3, 8, 10, 11])
);
assert_eq!(parse_cpu_list(" 12 "), Ok(vec![12]));
assert_eq!(parse_cpu_list(""), Ok(Vec::new()));
assert_eq!(parse_cpu_list(" \n"), Ok(Vec::new()));
assert_eq!(parse_cpu_list("5,1-2,1"), Ok(vec![1, 2, 5]));
}
#[test]
fn malformed_cpu_lists_are_rejected_not_panicked_on() {
assert!(parse_cpu_list("abc").is_err());
assert!(parse_cpu_list("0-").is_err());
assert!(parse_cpu_list("-3").is_err());
assert!(parse_cpu_list("3-1").is_err());
assert!(parse_cpu_list("0,,3").is_err());
assert!(parse_cpu_list("0-11 12").is_err());
assert!(parse_cpu_list("99999999999999999999").is_err());
assert!(parse_cpu_list(&format!("{}", MAX_CPUS)).is_err());
assert!(parse_cpu_list(&format!("0-{}", MAX_CPUS)).is_err());
assert!(parse_cpu_list(&format!("{}", MAX_CPUS - 1)).is_ok());
}
#[test]
fn cpu_lists_round_trip_through_the_formatter() {
for text in ["0-11", "0,5", "0-3,8,10-11", "12"] {
let parsed = parse_cpu_list(text).expect("parse");
assert_eq!(format_cpu_list(&parsed), text, "round trip of {text:?}");
}
assert_eq!(format_cpu_list(&[]), "-");
}
#[test]
fn reference_machine_yields_the_expected_smt_primaries() {
let tmp = reference_sysfs();
let topo = Topology::parse_at(tmp.path(), CpuMask::up_to(22)).expect("parse");
assert!(topo.is_hybrid());
assert!(topo.pins());
assert_eq!(topo.performance_cpus(), &(0..=11).collect::<Vec<_>>()[..]);
assert_eq!(topo.efficiency_cpus(), &(12..=21).collect::<Vec<_>>()[..]);
assert_eq!(
topo.l3_efficiency_cpus(),
&(12..=19).collect::<Vec<_>>()[..]
);
assert_eq!(topo.compute_cpus(), [0, 1, 3, 6, 8, 10]);
assert_eq!(topo.reactor_cpu(), Some(12));
assert_eq!(topo.shard_count(), 6);
}
#[test]
fn topology_display_is_readable_for_the_cli() {
let tmp = reference_sysfs();
let topo = Topology::parse_at(tmp.path(), CpuMask::up_to(22)).expect("parse");
let text = topo.to_string();
assert!(text.contains("hybrid"), "{text}");
assert!(text.contains("P 0-11"), "{text}");
assert!(text.contains("E 12-21 (L3 12-19)"), "{text}");
assert!(text.contains("compute 0-1,3,6,8,10"), "{text}");
assert!(text.contains("reactor 12"), "{text}");
assert!(text.contains("pinning on"), "{text}");
}
#[test]
fn non_hybrid_machines_degrade_to_no_pinning() {
let tmp = uniform_sysfs();
let topo = Topology::parse_at(tmp.path(), CpuMask::up_to(8)).expect("parse");
assert!(!topo.is_hybrid());
assert!(!topo.pins());
assert!(topo.compute_cpus().is_empty());
assert!(topo.performance_cpus().is_empty());
assert!(topo.efficiency_cpus().is_empty());
assert_eq!(topo.reactor_cpu(), None);
assert!(topo.shard_count() >= 1);
assert!(topo.shard_count() <= 8);
}
#[test]
fn missing_sysfs_degrades_instead_of_failing() {
let tmp = TempDir::new("absent");
let root = tmp.path().join("does-not-exist");
let topo = Topology::detect_at(&root, CpuMask::up_to(4));
assert!(!topo.pins());
assert!(topo.compute_cpus().is_empty());
assert_eq!(topo.reactor_cpu(), None);
assert!(topo.shard_count() >= 1);
}
#[test]
fn a_malformed_sibling_list_falls_through_to_the_other_spelling() {
let tmp = reference_sysfs();
write_file(
tmp.path(),
"devices/system/cpu/cpu3/topology/thread_siblings_list",
"three-and-four\n",
);
write_file(
tmp.path(),
"devices/system/cpu/cpu3/topology/core_cpus_list",
"3,4\n",
);
let topo = Topology::parse_at(tmp.path(), CpuMask::up_to(22)).expect("parse");
assert!(topo.pins());
assert_eq!(topo.compute_cpus(), [0, 1, 3, 6, 8, 10]);
}
#[test]
fn unknowable_smt_siblings_disable_pinning_rather_than_double_booking() {
for broken in ["three-and-four\n", ""] {
let tmp = reference_sysfs();
for name in ["thread_siblings_list", "core_cpus_list"] {
write_file(
tmp.path(),
&format!("devices/system/cpu/cpu3/topology/{name}"),
broken,
);
}
let topo = Topology::parse_at(tmp.path(), CpuMask::up_to(22)).expect("parse");
assert!(!topo.pins(), "{broken:?} still pinned: {topo}");
assert!(topo.compute_cpus().is_empty(), "{topo}");
assert!(topo.is_hybrid());
assert_eq!(topo.performance_cpus(), &(0..=11).collect::<Vec<_>>()[..]);
assert_eq!(topo.reactor_cpu(), Some(12));
assert!(topo.shard_count() >= 1);
}
}
#[test]
fn a_missing_topology_directory_disables_pinning() {
let tmp = reference_sysfs();
fs::remove_dir_all(tmp.path().join("devices/system/cpu/cpu0/topology"))
.expect("drop topology dir");
let topo = Topology::parse_at(tmp.path(), CpuMask::up_to(22)).expect("parse");
assert!(!topo.pins(), "{topo}");
assert!(topo.compute_cpus().is_empty(), "{topo}");
assert!(!Topology::detect_at(tmp.path(), CpuMask::up_to(22)).pins());
}
#[test]
fn malformed_pmu_cpu_list_fails_parse_and_degrades_detect() {
let tmp = reference_sysfs();
write_file(tmp.path(), "devices/cpu_core/cpus", "0-11,oops\n");
assert!(Topology::parse_at(tmp.path(), CpuMask::up_to(22)).is_err());
assert!(!Topology::detect_at(tmp.path(), CpuMask::up_to(22)).pins());
}
#[test]
fn topology_never_leaves_the_affinity_mask() {
let tmp = reference_sysfs();
let allowed: CpuMask = [0, 1, 2, 3, 13].into_iter().collect();
let topo = Topology::parse_at(tmp.path(), allowed.clone()).expect("parse");
assert_eq!(topo.compute_cpus(), [0, 1, 3]);
assert_eq!(topo.performance_cpus(), [0, 1, 2, 3]);
assert_eq!(topo.efficiency_cpus(), [13]);
assert_eq!(topo.reactor_cpu(), Some(13));
for cpu in topo
.compute_cpus()
.iter()
.chain(topo.performance_cpus())
.chain(topo.efficiency_cpus())
.chain(topo.l3_efficiency_cpus())
.chain(topo.reactor_cpu().iter())
{
assert!(allowed.contains(*cpu), "cpu {cpu} escaped the mask");
}
}
#[test]
fn a_single_allowed_cpu_disables_pinning() {
let tmp = reference_sysfs();
let topo = Topology::parse_at(tmp.path(), [3].into_iter().collect()).expect("parse");
assert!(topo.is_hybrid());
assert!(!topo.pins());
assert!(topo.compute_cpus().is_empty());
assert_eq!(topo.shard_count(), 1);
}
#[test]
fn an_empty_affinity_mask_is_an_error_not_a_panic() {
let tmp = reference_sysfs();
let err = Topology::parse_at(tmp.path(), CpuMask::new()).expect_err("must fail");
assert!(matches!(err, TopologyError::NoCpus), "{err:?}");
}
#[test]
fn hybrid_without_any_l3_ecore_leaves_the_reactor_unpinned() {
let tmp = reference_sysfs();
for cpu in 12..=19 {
let dir = tmp
.path()
.join(format!("devices/system/cpu/cpu{cpu}/cache/index3"));
fs::remove_dir_all(&dir).expect("drop l3 index");
}
let topo = Topology::parse_at(tmp.path(), CpuMask::up_to(22)).expect("parse");
assert!(topo.l3_efficiency_cpus().is_empty());
assert_eq!(topo.reactor_cpu(), None);
assert_eq!(topo.compute_cpus(), [0, 1, 3, 6, 8, 10]);
}
#[test]
fn detect_on_this_host_never_panics() {
let topo = Topology::detect();
assert!(topo.shard_count() >= 1);
for cpu in topo.compute_cpus() {
assert!(topo.allowed().contains(*cpu));
}
if let Some(cpu) = topo.reactor_cpu() {
assert!(topo.allowed().contains(cpu));
}
let host_p_cores: Vec<usize> = fs::read_to_string("/sys/devices/cpu_core/cpus")
.ok()
.and_then(|text| parse_cpu_list(text.trim()).ok())
.unwrap_or_default();
let hybrid_host =
!host_p_cores.is_empty() && Path::new("/sys/devices/cpu_atom/cpus").exists();
let usable_p = host_p_cores
.iter()
.filter(|cpu| topo.allowed().contains(**cpu))
.count();
if hybrid_host && usable_p >= 4 {
assert!(topo.is_hybrid(), "hybrid host reported as uniform: {topo}");
assert!(topo.pins(), "hybrid host left unpinned: {topo}");
assert!(topo.compute_cpus().len() >= 2, "{topo}");
assert!(topo.reactor_cpu().is_some(), "{topo}");
}
}
#[test]
fn cpu_masks_intersect() {
let a: CpuMask = [0, 1, 3, 6, 8, 10].into_iter().collect();
let b: CpuMask = [1, 2, 3, 4, 10].into_iter().collect();
assert_eq!(a.intersect(&b).to_vec(), [1, 3, 10]);
assert_eq!(b.intersect(&a), a.intersect(&b));
assert!(a.intersect(&CpuMask::new()).is_empty());
assert_eq!(a.len(), 6);
assert!(a.contains(8));
assert!(!a.contains(9));
}
#[test]
fn cpu_masks_refuse_ids_that_would_overflow_the_bit_array() {
let mut mask = CpuMask::new();
assert!(mask.insert(MAX_CPUS - 1));
assert!(!mask.insert(MAX_CPUS));
assert!(!mask.insert(usize::MAX));
assert_eq!(mask.to_vec(), [MAX_CPUS - 1]);
assert_eq!(CpuMask::up_to(usize::MAX).len(), MAX_CPUS);
}
#[test]
fn pinning_out_of_range_is_a_typed_error() {
let err = pin_current_thread(MAX_CPUS).expect_err("must fail");
assert!(matches!(err, PinError::OutOfRange { .. }), "{err:?}");
let err = pin_current_thread_to(&CpuMask::new()).expect_err("must fail");
assert!(matches!(err, PinError::EmptyMask), "{err:?}");
}
#[test]
fn pinning_outside_the_affinity_mask_is_refused() {
let Some(allowed) = CpuMask::current() else {
return; };
let outside = (0..MAX_CPUS).find(|c| !allowed.contains(*c));
if let Some(cpu) = outside {
let err = pin_current_thread(cpu).expect_err("must fail");
assert!(matches!(err, PinError::NotAllowed { .. }), "{err:?}");
}
}
#[test]
fn shard_ranges_tile_the_row_space_exactly_once() {
for rows in [0usize, 1, 2, 3, 7, 8, 64, 127, 1000, 4096] {
for shards in [1usize, 2, 3, 5, 6, 8, 16] {
let mut next = 0;
let mut seen = 0;
for i in 0..shards {
let r = shard_range(rows, shards, i);
assert_eq!(r.start, next, "rows={rows} shards={shards} i={i}");
assert!(r.end >= r.start);
next = r.end;
seen += r.len();
}
assert_eq!(next, rows, "rows={rows} shards={shards}");
assert_eq!(seen, rows, "rows={rows} shards={shards}");
}
}
}
#[test]
fn shard_ranges_are_balanced_and_deterministic() {
let got: Vec<_> = (0..4).map(|i| shard_range(10, 4, i)).collect();
assert_eq!(got, vec![0..3, 3..6, 6..8, 8..10]);
for _ in 0..8 {
assert_eq!(shard_range(10, 4, 1), 3..6);
}
assert_eq!(shard_range(10, 0, 0), 0..0);
assert_eq!(shard_range(10, 4, 4), 0..0);
}
fn test_pool(shards: usize) -> ComputePool {
ComputePool::with_topology(
Topology::unpinned(CpuMask::up_to(shards.max(1))),
PoolConfig {
shards: Some(shards),
placement: Placement::Off,
pin_caller: false,
inline_caller: true,
..PoolConfig::default()
},
)
}
#[test]
fn every_row_is_visited_exactly_once() {
let mut pool = test_pool(4);
assert_eq!(pool.shards(), 4);
let rows = 1000;
let visits: Vec<AtomicU32> = (0..rows).map(|_| AtomicU32::new(0)).collect();
pool.run(rows, |shard| {
for row in shard.rows.clone() {
visits[row].fetch_add(1, Ordering::Relaxed);
}
});
for (row, v) in visits.iter().enumerate() {
assert_eq!(v.load(Ordering::Relaxed), 1, "row {row}");
}
}
#[test]
fn shards_are_disjoint_and_cover_the_job() {
let mut pool = test_pool(6);
let seen: Mutex<Vec<Range<usize>>> = Mutex::new(Vec::new());
pool.run(97, |shard| {
assert_eq!(shard.count, 6);
assert!(!shard.is_empty());
assert_eq!(shard.len(), shard.rows.len());
seen.lock().expect("lock").push(shard.rows.clone());
});
let mut ranges = seen.into_inner().expect("into_inner");
ranges.sort_by_key(|r| r.start);
assert_eq!(ranges.len(), 6);
let mut next = 0;
for r in &ranges {
assert_eq!(r.start, next);
next = r.end;
}
assert_eq!(next, 97);
}
#[test]
fn parallel_results_match_the_single_threaded_reference() {
let rows = 2003;
let weights: Vec<f32> = (0..rows).map(|i| (i as f32).mul_add(0.25, -3.0)).collect();
let mut expected = vec![0.0f32; rows];
for (i, out) in expected.iter_mut().enumerate() {
*out = weights[i].mul_add(weights[i], weights[i]);
}
for shards in [1usize, 2, 3, 7] {
let mut pool = test_pool(shards);
let mut got = vec![0.0f32; rows];
pool.scatter(&mut got, |shard, chunk| {
for (k, slot) in chunk.iter_mut().enumerate() {
let w = weights[shard.rows.start + k];
*slot = w.mul_add(w, w);
}
});
assert_eq!(got, expected, "shards={shards}");
}
}
#[test]
fn repeated_jobs_reuse_the_same_threads() {
let mut pool = test_pool(4);
let rows = 128;
let total = AtomicUsize::new(0);
for _ in 0..2000 {
pool.run(rows, |shard| {
total.fetch_add(shard.rows.len(), Ordering::Relaxed);
});
}
assert_eq!(total.load(Ordering::Relaxed), 2000 * rows);
}
#[test]
fn a_straggler_cannot_consume_the_next_jobs_park_flag() {
let finished = Arc::new(AtomicU32::new(0));
let stalled = Arc::new(AtomicU32::new(0));
let saw_flag = Arc::new(AtomicU32::new(0));
let flag_recorded = Arc::new(AtomicU32::new(0));
let flag_survived = Arc::new(AtomicU32::new(0));
let scenario = {
let (finished, stalled, saw_flag, flag_recorded, flag_survived) = (
Arc::clone(&finished),
Arc::clone(&stalled),
Arc::clone(&saw_flag),
Arc::clone(&flag_recorded),
Arc::clone(&flag_survived),
);
std::thread::spawn(move || {
let mut pool = test_pool(3);
pool.shared.probe.arm_worker();
pool.run(300, |shard| {
if shard.index == 0 {
std::thread::sleep(Duration::from_millis(100));
}
});
pool.shared.probe.arm_lead();
pool.run(300, |shard| {
if shard.index != 0 {
std::thread::sleep(Duration::from_millis(150));
}
});
stalled.store(
u32::from(pool.shared.probe.straggler_went_through_window()),
Ordering::SeqCst,
);
saw_flag.store(
u32::from(pool.shared.probe.worker_saw_flag()),
Ordering::SeqCst,
);
let (recorded, survived) = pool.shared.probe.flag_survived_straggler();
flag_recorded.store(u32::from(recorded), Ordering::SeqCst);
flag_survived.store(u32::from(survived), Ordering::SeqCst);
finished.store(1, Ordering::SeqCst);
})
};
let deadline = Instant::now() + Duration::from_secs(30);
while finished.load(Ordering::SeqCst) == 0 && Instant::now() < deadline {
std::thread::sleep(Duration::from_millis(5));
}
assert_eq!(
finished.load(Ordering::SeqCst),
1,
"the compute pool deadlocked: a straggler from the previous job \
consumed the park flag armed for the current one"
);
scenario.join().expect("scenario thread");
assert_eq!(
stalled.load(Ordering::SeqCst),
1,
"the probe never stalled a worker in the wake window; the \
interleaving under test was not exercised"
);
assert_eq!(
saw_flag.load(Ordering::SeqCst),
1,
"the straggler did not observe the submitter's park flag, so it \
was never in a position to consume it; the interleaving under \
test was not exercised"
);
assert_eq!(
flag_recorded.load(Ordering::SeqCst),
1,
"the submitter never reached its park window with a straggler \
held in the wake window; the interleaving under test was not \
exercised"
);
assert_eq!(
flag_survived.load(Ordering::SeqCst),
1,
"a straggler from the previous job cleared the park flag armed \
for the current one; the submitter is about to park with \
`lead_parked == 0` and only `BARRIER_POLL` will get it out"
);
}
#[test]
fn tiny_and_empty_jobs_run_inline() {
let mut pool = test_pool(6);
let calls = AtomicUsize::new(0);
pool.run(0, |_| {
calls.fetch_add(1, Ordering::Relaxed);
});
assert_eq!(calls.load(Ordering::Relaxed), 0);
let shards = Mutex::new(Vec::new());
pool.run(3, |shard| {
shards.lock().expect("lock").push(shard);
});
let shards = shards.into_inner().expect("into_inner");
assert_eq!(shards.len(), 1);
assert_eq!(shards[0].count, 1);
assert_eq!(shards[0].rows, 0..3);
}
#[test]
fn a_pool_with_one_shard_still_runs_the_job() {
let mut pool = test_pool(1);
assert_eq!(pool.shards(), 1);
let sum = AtomicUsize::new(0);
pool.run(10, |shard| {
sum.fetch_add(shard.rows.len(), Ordering::Relaxed);
});
assert_eq!(sum.load(Ordering::Relaxed), 10);
}
#[test]
fn borrowed_non_static_data_is_usable_from_the_pool() {
let mut pool = test_pool(4);
let weights = vec![2.0f32; 512];
let bias = 0.5f32;
let mut out = vec![0.0f32; 512];
pool.scatter(&mut out, |shard, chunk| {
for (k, slot) in chunk.iter_mut().enumerate() {
*slot = weights[shard.rows.start + k] + bias;
}
});
assert!(out.iter().all(|v| (*v - 2.5).abs() < 1e-6));
drop(weights);
}
#[test]
fn a_panicking_shard_neither_deadlocks_nor_poisons_the_pool() {
let mut pool = test_pool(4);
let hook = std::panic::take_hook();
std::panic::set_hook(Box::new(|_| {}));
let result = catch_unwind(AssertUnwindSafe(|| {
pool.run(400, |shard| {
assert!(shard.index != 2, "boom");
});
}));
let caller = catch_unwind(AssertUnwindSafe(|| {
pool.run(400, |shard| {
assert!(shard.index != 0, "boom-lead");
});
}));
std::panic::set_hook(hook);
assert!(result.is_err(), "worker panic must propagate");
assert!(caller.is_err(), "caller panic must propagate");
let sum = AtomicUsize::new(0);
pool.run(400, |shard| {
sum.fetch_add(shard.rows.len(), Ordering::Relaxed);
});
assert_eq!(sum.load(Ordering::Relaxed), 400);
}
#[test]
fn chunk_counts_are_bounded_by_rows_and_never_zero() {
assert_eq!(chunk_count(1000, 6, 4), 24);
assert_eq!(chunk_count(10, 6, 4), 10);
assert_eq!(chunk_count(1, 6, 4), 1);
assert_eq!(chunk_count(0, 6, 4), 1);
assert_eq!(chunk_count(1000, 0, 4), 1);
assert_eq!(chunk_count(1000, usize::MAX, 4), 1000);
}
#[test]
fn every_chunk_runs_once_with_the_rows_its_index_names() {
for shards in [1usize, 2, 3, 6] {
let mut pool = test_pool(shards);
for rows in [1usize, 5, 23, 24, 25, 1000] {
let count = pool.chunks(rows);
let runs: Vec<AtomicU32> = (0..count).map(|_| AtomicU32::new(0)).collect();
let visits: Vec<AtomicU32> = (0..rows).map(|_| AtomicU32::new(0)).collect();
pool.run_chunks(rows, |chunk| {
assert_eq!(chunk.count, count);
assert_eq!(chunk.rows, shard_range(rows, count, chunk.index));
runs[chunk.index].fetch_add(1, Ordering::Relaxed);
for row in chunk.rows {
visits[row].fetch_add(1, Ordering::Relaxed);
}
});
for (i, r) in runs.iter().enumerate() {
assert_eq!(
r.load(Ordering::Relaxed),
1,
"{shards} shards, {rows} rows, chunk {i}"
);
}
for (row, v) in visits.iter().enumerate() {
assert_eq!(
v.load(Ordering::Relaxed),
1,
"{shards} shards, {rows} rows, row {row}"
);
}
}
}
}
#[test]
fn chunked_scatter_matches_the_single_threaded_reference() {
let rows = 2003;
let weights: Vec<f32> = (0..rows).map(|i| (i as f32).mul_add(0.25, -3.0)).collect();
let expected: Vec<f32> = weights.iter().map(|w| w.mul_add(*w, *w)).collect();
for shards in [1usize, 2, 3, 7] {
let mut pool = test_pool(shards);
let mut got = vec![0.0f32; rows];
pool.scatter_chunks(&mut got, |chunk, piece| {
assert_eq!(piece.len(), chunk.rows.len());
for (k, slot) in piece.iter_mut().enumerate() {
let w = weights[chunk.rows.start + k];
*slot = w.mul_add(w, w);
}
});
assert_eq!(got, expected, "{shards} shards");
}
}
#[test]
fn chunked_and_sharded_jobs_interleave_on_one_pool() {
let mut pool = test_pool(4);
for round in 0..200 {
let rows = 50 + round % 13;
let sum = AtomicUsize::new(0);
if round % 3 == 0 {
pool.run(rows, |shard| {
sum.fetch_add(shard.rows.len(), Ordering::Relaxed);
});
} else {
pool.run_chunks(rows, |chunk| {
sum.fetch_add(chunk.rows.len(), Ordering::Relaxed);
});
}
assert_eq!(sum.load(Ordering::Relaxed), rows, "round {round}");
}
}
#[test]
fn a_stalled_thread_leaves_the_rest_of_the_job_to_the_others() {
let mut pool = test_pool(4);
let rows = 400;
let count = pool.chunks(rows);
assert!(count > 4, "the test needs more chunks than threads");
let ran: Mutex<Vec<(std::thread::ThreadId, usize)>> = Mutex::new(Vec::new());
pool.run_chunks(rows, |chunk| {
if chunk.index == 0 {
std::thread::sleep(Duration::from_millis(200));
}
ran.lock()
.expect("lock")
.push((std::thread::current().id(), chunk.index));
});
let ran = ran.into_inner().expect("into_inner");
assert_eq!(ran.len(), count);
let stalled = ran.iter().find(|(_, i)| *i == 0).expect("chunk 0 ran").0;
let by_stalled: Vec<usize> = ran
.iter()
.filter(|(t, _)| *t == stalled)
.map(|(_, i)| *i)
.collect();
assert_eq!(by_stalled, vec![0], "the stalled thread ran {by_stalled:?}");
}
#[test]
fn the_caller_runs_chunks_and_says_so() {
let mut pool = test_pool(4);
let caller = std::thread::current().id();
let rows = 4000;
pool.run_chunks(rows, |chunk| {
let here = std::thread::current().id() == caller;
assert_eq!(chunk.on_caller, here, "chunk {}", chunk.index);
});
let mut solo = test_pool(1);
solo.run_chunks(rows, |chunk| {
assert!(chunk.on_caller);
assert_eq!(chunk.rows, 0..rows);
});
}
#[test]
fn a_panicking_chunk_neither_deadlocks_nor_poisons_the_pool() {
let mut pool = test_pool(4);
let hook = std::panic::take_hook();
std::panic::set_hook(Box::new(|_| {}));
let attempted = AtomicUsize::new(0);
let result = catch_unwind(AssertUnwindSafe(|| {
pool.run_chunks(400, |chunk| {
attempted.fetch_add(1, Ordering::Relaxed);
assert!(chunk.index != 5, "boom");
});
}));
std::panic::set_hook(hook);
assert!(result.is_err(), "a chunk panic must propagate");
assert_eq!(
attempted.load(Ordering::Relaxed),
pool.chunks(400),
"every chunk is still attempted"
);
let sum = AtomicUsize::new(0);
pool.run_chunks(400, |chunk| {
sum.fetch_add(chunk.rows.len(), Ordering::Relaxed);
});
assert_eq!(sum.load(Ordering::Relaxed), 400);
}
#[test]
fn dropping_the_pool_joins_every_worker() {
let pool = test_pool(6);
let watch = Arc::clone(&pool.shared);
assert_eq!(Arc::strong_count(&watch), 7); drop(pool);
assert_eq!(Arc::strong_count(&watch), 1);
}
#[test]
fn pools_can_be_built_and_torn_down_repeatedly() {
for shards in 1..=8 {
let mut pool = test_pool(shards);
let sum = AtomicUsize::new(0);
pool.run(64, |shard| {
sum.fetch_add(shard.rows.len(), Ordering::Relaxed);
});
assert_eq!(sum.load(Ordering::Relaxed), 64);
}
}
#[test]
fn a_pool_without_an_inline_caller_uses_only_workers() {
let mut pool = ComputePool::with_topology(
Topology::unpinned(CpuMask::up_to(4)),
PoolConfig {
shards: Some(4),
placement: Placement::Off,
pin_caller: false,
inline_caller: false,
..PoolConfig::default()
},
);
assert_eq!(pool.shards(), 4);
let caller = std::thread::current().id();
let elsewhere = AtomicUsize::new(0);
pool.run(400, |shard| {
if std::thread::current().id() != caller {
elsewhere.fetch_add(shard.rows.len(), Ordering::Relaxed);
}
});
assert_eq!(elsewhere.load(Ordering::Relaxed), 400);
}
#[test]
fn debug_output_names_the_pool_shape() {
let pool = test_pool(3);
let text = format!("{pool:?}");
assert!(text.contains("shards: 3"), "{text}");
assert!(text.contains("workers: 2"), "{text}");
}
#[test]
#[ignore = "timing probe, not a correctness gate"]
fn wake_latency_probe() {
use std::time::Instant;
let mut pool = ComputePool::new();
let shards = pool.shards();
let rows = shards * 8;
let mut samples = Vec::with_capacity(20_000);
for _ in 0..2_000 {
pool.run(rows, |_| {});
}
for _ in 0..20_000 {
let start = Instant::now();
pool.run(rows, |_| {});
samples.push(start.elapsed().as_nanos() as u64);
}
samples.sort_unstable();
let pct = |p: usize| samples[samples.len() * p / 100];
eprintln!(
"shards={shards} barrier ns: p50={} p90={} p99={} max={}",
pct(50),
pct(90),
pct(99),
samples[samples.len() - 1]
);
}
}