use core::sync::atomic::{AtomicU32, Ordering};
use std::sync::OnceLock;
use std::time::Duration;
use crate::sched::arena::global_local_arena;
use crate::sched::io_pool::global_io_pool;
static SPLIT_MULTIPLIER: AtomicU32 = AtomicU32::new(2);
use core::sync::atomic::AtomicU64;
static LEAF_TIME_SUM_NS: AtomicU64 = AtomicU64::new(0);
static LEAF_TIME_SUMSQ: AtomicU64 = AtomicU64::new(0);
static LEAF_COUNT: AtomicU64 = AtomicU64::new(0);
#[inline]
pub fn record_leaf_time_ns(nanos: u64) {
LEAF_TIME_SUM_NS.fetch_add(nanos, Ordering::Relaxed);
let scaled = nanos >> 8;
LEAF_TIME_SUMSQ.fetch_add(scaled.saturating_mul(scaled), Ordering::Relaxed);
LEAF_COUNT.fetch_add(1, Ordering::Relaxed);
}
#[inline]
pub fn record_leaf_batch(sum_ns: u64, sumsq_scaled: u64, count: u64) {
LEAF_TIME_SUM_NS.fetch_add(sum_ns, Ordering::Relaxed);
LEAF_TIME_SUMSQ.fetch_add(sumsq_scaled, Ordering::Relaxed);
let prior = LEAF_COUNT.fetch_add(count, Ordering::Relaxed);
let new_total = prior.wrapping_add(count);
if (prior / AUTO_CLASSIFY_QUANTUM) != (new_total / AUTO_CLASSIFY_QUANTUM) {
crate::sched::adaptive_profile::tick_auto_classify();
}
}
pub const AUTO_CLASSIFY_QUANTUM: u64 = 16;
#[cfg(test)]
static TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
#[cfg(test)]
pub fn acquire_test_lock() -> std::sync::MutexGuard<'static, ()> {
TEST_LOCK.lock().unwrap_or_else(|p| p.into_inner())
}
#[inline]
pub fn reset_leaf_stats() {
LEAF_TIME_SUM_NS.store(0, Ordering::Relaxed);
LEAF_TIME_SUMSQ.store(0, Ordering::Relaxed);
LEAF_COUNT.store(0, Ordering::Relaxed);
}
#[derive(Copy, Clone, Debug)]
pub struct LeafStats {
pub count: u64,
pub sum_ns: u64,
pub sumsq_scaled: u64,
}
#[inline]
pub fn snapshot_leaf_stats() -> LeafStats {
LeafStats {
count: LEAF_COUNT.load(Ordering::Relaxed),
sum_ns: LEAF_TIME_SUM_NS.load(Ordering::Relaxed),
sumsq_scaled: LEAF_TIME_SUMSQ.load(Ordering::Relaxed),
}
}
#[inline]
pub fn observed_mean_leaf_ns() -> Option<u64> {
let stats = snapshot_leaf_stats();
if stats.count < 4 {
return None;
}
Some(stats.sum_ns / stats.count)
}
pub fn leaf_cv_squared_per_mille(stats: LeafStats) -> Option<u64> {
if stats.count < 4 {
return None;
}
let n = stats.count;
let mean_scaled = (stats.sum_ns >> 8) / n;
if mean_scaled == 0 {
return Some(0);
}
let sumsq_per_n = stats.sumsq_scaled / n;
let mean_sq = mean_scaled.saturating_mul(mean_scaled);
let var = sumsq_per_n.saturating_sub(mean_sq);
Some(var.saturating_mul(1000) / mean_sq.max(1))
}
#[inline]
pub fn split_multiplier() -> u32 {
SPLIT_MULTIPLIER.load(Ordering::Relaxed)
}
pub fn set_split_multiplier(value: u32) {
let clamped = value.clamp(1, 8);
SPLIT_MULTIPLIER.store(clamped, Ordering::Relaxed);
}
pub fn spawn_observer() {
static STARTED: OnceLock<()> = OnceLock::new();
STARTED.get_or_init(|| {
if global_io_pool().is_none() {
return;
}
schedule_next_sample();
});
}
fn schedule_next_sample() {
let Some(pool) = global_io_pool() else { return };
pool.submit(|| {
std::thread::sleep(Duration::from_millis(200));
let new_mult = sample_and_compute();
SPLIT_MULTIPLIER.store(new_mult, Ordering::Relaxed);
schedule_next_sample();
});
}
pub fn sample_and_compute() -> u32 {
let arena = global_local_arena();
let mut total_pops: u64 = 0;
let mut total_steals: u64 = 0;
for stats in arena.iter_worker_stats() {
total_pops = total_pops.saturating_add(
stats.local_pops.load(Ordering::Relaxed),
);
total_steals = total_steals.saturating_add(
stats.peer_steal_hits.load(Ordering::Relaxed),
);
}
let total = total_pops.saturating_add(total_steals);
if total < 100 {
return SPLIT_MULTIPLIER.load(Ordering::Relaxed);
}
let steal_per_mille: u64 = (total_steals * 1000) / total;
let from_steal = if steal_per_mille >= 300 {
4u32
} else if steal_per_mille >= 50 {
2u32
} else {
1u32
};
let stats = snapshot_leaf_stats();
let from_variance = match leaf_cv_squared_per_mille(stats) {
Some(cv2) if cv2 >= 500 => 4u32,
Some(cv2) if cv2 >= 50 => 2u32,
Some(_) => 1u32,
None => from_steal,
};
reset_leaf_stats();
from_steal.max(from_variance).min(8)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn split_multiplier_starts_at_baseline_two() {
let m = split_multiplier();
assert!((1..=8).contains(&m), "multiplier should be in [1, 8]");
}
#[test]
fn set_split_multiplier_clamps_to_valid_range() {
set_split_multiplier(100);
assert_eq!(split_multiplier(), 8);
set_split_multiplier(0);
assert_eq!(split_multiplier(), 1);
set_split_multiplier(2);
}
#[test]
fn spawn_observer_noop_when_pool_disabled() {
let t0 = std::time::Instant::now();
spawn_observer();
assert!(
t0.elapsed() < Duration::from_millis(50),
"spawn_observer should return immediately"
);
}
#[test]
fn leaf_stats_record_and_reset() {
reset_leaf_stats();
record_leaf_time_ns(1_000);
record_leaf_time_ns(2_000);
record_leaf_time_ns(3_000);
let s = snapshot_leaf_stats();
assert_eq!(s.count, 3);
assert_eq!(s.sum_ns, 6_000);
reset_leaf_stats();
let s2 = snapshot_leaf_stats();
assert_eq!(s2.count, 0);
assert_eq!(s2.sum_ns, 0);
assert_eq!(s2.sumsq_scaled, 0);
}
#[test]
fn leaf_cv_squared_handles_insufficient_samples() {
reset_leaf_stats();
for _ in 0..3 {
record_leaf_time_ns(1_000);
}
let s = snapshot_leaf_stats();
assert!(leaf_cv_squared_per_mille(s).is_none(),
"fewer than 4 leaves should return None");
reset_leaf_stats();
}
#[test]
fn leaf_cv_squared_zero_for_uniform_leaves() {
reset_leaf_stats();
for _ in 0..8 {
record_leaf_time_ns(10_000);
}
let s = snapshot_leaf_stats();
let cv2 = leaf_cv_squared_per_mille(s).unwrap();
assert!(cv2 < 20, "uniform leaves should have cv^2 ~ 0; got {cv2}");
reset_leaf_stats();
}
#[test]
fn leaf_cv_squared_high_for_spread_leaves() {
reset_leaf_stats();
for _ in 0..4 {
record_leaf_time_ns(1_000);
}
for _ in 0..4 {
record_leaf_time_ns(100_000);
}
let s = snapshot_leaf_stats();
let cv2 = leaf_cv_squared_per_mille(s).unwrap();
assert!(cv2 >= 500, "spread leaves should have cv^2 >= 500; got {cv2}");
reset_leaf_stats();
}
#[test]
fn sample_returns_current_multiplier_when_no_data() {
for stats in global_local_arena().iter_worker_stats() {
stats.local_pops.store(0, Ordering::Relaxed);
stats.peer_steal_hits.store(0, Ordering::Relaxed);
stats.peer_steal_misses.store(0, Ordering::Relaxed);
}
set_split_multiplier(3);
let computed = sample_and_compute();
assert_eq!(computed, 3);
set_split_multiplier(2);
}
}