use core::sync::atomic::{AtomicU32, Ordering};
use std::sync::OnceLock;
use std::time::Duration;
use crate::sched::arena::global_local_arena;
use crate::sched::io_pool::global_io_pool;
static SPLIT_MULTIPLIER: AtomicU32 = AtomicU32::new(2);
use core::sync::atomic::AtomicU64;
static LEAF_TIME_SUM_NS: AtomicU64 = AtomicU64::new(0);
static LEAF_TIME_SUMSQ: AtomicU64 = AtomicU64::new(0);
static LEAF_COUNT: AtomicU64 = AtomicU64::new(0);
static LEAF_ITEMS: AtomicU64 = AtomicU64::new(0);
static LEAF_SUMSQ_PER_ITEM: AtomicU64 = AtomicU64::new(0);
#[inline]
pub fn record_leaf_time_ns(nanos: u64) {
LEAF_TIME_SUM_NS.fetch_add(nanos, Ordering::Relaxed);
let scaled = nanos >> 8;
LEAF_TIME_SUMSQ.fetch_add(scaled.saturating_mul(scaled), Ordering::Relaxed);
LEAF_COUNT.fetch_add(1, Ordering::Relaxed);
}
#[inline]
pub fn record_leaf_batch(
sum_ns: u64,
sumsq_scaled: u64,
count: u64,
items: u64,
sumsq_per_item: u64,
) {
LEAF_TIME_SUM_NS.fetch_add(sum_ns, Ordering::Relaxed);
LEAF_TIME_SUMSQ.fetch_add(sumsq_scaled, Ordering::Relaxed);
LEAF_ITEMS.fetch_add(items, Ordering::Relaxed);
LEAF_SUMSQ_PER_ITEM.fetch_add(sumsq_per_item, Ordering::Relaxed);
let prior = LEAF_COUNT.fetch_add(count, Ordering::Relaxed);
let new_total = prior.wrapping_add(count);
if (prior / AUTO_CLASSIFY_QUANTUM) != (new_total / AUTO_CLASSIFY_QUANTUM) {
crate::sched::adaptive_profile::tick_auto_classify();
}
}
pub const AUTO_CLASSIFY_QUANTUM: u64 = 16;
#[inline]
pub fn reset_leaf_stats() {
LEAF_TIME_SUM_NS.store(0, Ordering::Relaxed);
LEAF_TIME_SUMSQ.store(0, Ordering::Relaxed);
LEAF_COUNT.store(0, Ordering::Relaxed);
LEAF_ITEMS.store(0, Ordering::Relaxed);
LEAF_SUMSQ_PER_ITEM.store(0, Ordering::Relaxed);
}
#[derive(Copy, Clone, Debug)]
pub struct LeafStats {
pub count: u64,
pub sum_ns: u64,
pub sumsq_scaled: u64,
pub items: u64,
pub sumsq_per_item: u64,
}
#[inline]
pub fn snapshot_leaf_stats() -> LeafStats {
LeafStats {
count: LEAF_COUNT.load(Ordering::Relaxed),
sum_ns: LEAF_TIME_SUM_NS.load(Ordering::Relaxed),
sumsq_scaled: LEAF_TIME_SUMSQ.load(Ordering::Relaxed),
items: LEAF_ITEMS.load(Ordering::Relaxed),
sumsq_per_item: LEAF_SUMSQ_PER_ITEM.load(Ordering::Relaxed),
}
}
#[inline]
pub fn observed_mean_leaf_ns() -> Option<u64> {
let stats = snapshot_leaf_stats();
if stats.count < 4 {
return None;
}
Some(stats.sum_ns / stats.count)
}
#[inline]
pub fn observed_per_item_ns(stats: LeafStats) -> Option<u64> {
if stats.count < 4 || stats.items == 0 {
return None;
}
Some(stats.sum_ns / stats.items)
}
pub fn per_item_cv_squared_per_mille(stats: LeafStats) -> Option<u64> {
if stats.count < 4 || stats.items == 0 {
return None;
}
let mean_ns = stats.sum_ns / stats.items;
let mean_sq = ((mean_ns as u128).saturating_mul(mean_ns as u128) >> 16) as u64;
if mean_sq == 0 {
return None;
}
let spread = stats
.sumsq_per_item
.saturating_sub(mean_sq.saturating_mul(stats.items));
let var = spread / stats.items;
Some(var.saturating_mul(1000) / mean_sq)
}
pub fn leaf_cv_squared_per_mille(stats: LeafStats) -> Option<u64> {
if stats.count < 4 {
return None;
}
let n = stats.count;
let mean_scaled = (stats.sum_ns >> 8) / n;
if mean_scaled == 0 {
return None;
}
let sumsq_per_n = stats.sumsq_scaled / n;
let mean_sq = mean_scaled.saturating_mul(mean_scaled);
let var = sumsq_per_n.saturating_sub(mean_sq);
Some(var.saturating_mul(1000) / mean_sq.max(1))
}
#[inline]
pub fn split_multiplier() -> u32 {
SPLIT_MULTIPLIER.load(Ordering::Relaxed)
}
pub fn set_split_multiplier(value: u32) {
let clamped = value.clamp(1, 8);
SPLIT_MULTIPLIER.store(clamped, Ordering::Relaxed);
}
pub fn spawn_observer() {
static STARTED: OnceLock<()> = OnceLock::new();
STARTED.get_or_init(|| {
if global_io_pool().is_none() {
return;
}
schedule_next_sample();
});
}
fn schedule_next_sample() {
let Some(pool) = global_io_pool() else { return };
pool.submit(|| {
std::thread::sleep(Duration::from_millis(200));
let new_mult = sample_and_compute();
SPLIT_MULTIPLIER.store(new_mult, Ordering::Relaxed);
schedule_next_sample();
});
}
const MIN_WINDOW_EVENTS: u64 = 100;
fn multiplier_for_window(
total_pops: u64,
total_steals: u64,
cv2: Option<u64>,
current: u32,
) -> u32 {
let total = total_pops.saturating_add(total_steals);
if total < MIN_WINDOW_EVENTS {
return current;
}
let steal_per_mille: u64 = (total_steals * 1000) / total;
let from_steal = if steal_per_mille >= 300 {
4u32
} else if steal_per_mille >= 50 {
2u32
} else {
1u32
};
let from_variance = match cv2 {
Some(cv2) if cv2 >= 500 => 4u32,
Some(cv2) if cv2 >= 50 => 2u32,
Some(_) => 1u32,
None => from_steal,
};
from_steal.max(from_variance).min(8)
}
pub fn sample_and_compute() -> u32 {
let arena = global_local_arena();
let mut total_pops: u64 = 0;
let mut total_steals: u64 = 0;
for stats in arena.iter_worker_stats() {
total_pops = total_pops.saturating_add(
stats.local_pops.load(Ordering::Relaxed),
);
total_steals = total_steals.saturating_add(
stats.peer_steal_hits.load(Ordering::Relaxed),
);
}
let current = SPLIT_MULTIPLIER.load(Ordering::Relaxed);
if total_pops.saturating_add(total_steals) < MIN_WINDOW_EVENTS {
return current;
}
let cv2 = leaf_cv_squared_per_mille(snapshot_leaf_stats());
let mult = multiplier_for_window(total_pops, total_steals, cv2, current);
reset_leaf_stats();
mult
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn split_multiplier_starts_at_baseline_two() {
let m = split_multiplier();
assert!((1..=8).contains(&m), "multiplier should be in [1, 8]");
}
#[test]
fn set_split_multiplier_clamps_to_valid_range() {
set_split_multiplier(100);
assert_eq!(split_multiplier(), 8);
set_split_multiplier(0);
assert_eq!(split_multiplier(), 1);
set_split_multiplier(2);
}
#[test]
fn spawn_observer_noop_when_pool_disabled() {
let t0 = std::time::Instant::now();
spawn_observer();
assert!(
t0.elapsed() < Duration::from_millis(50),
"spawn_observer should return immediately"
);
}
fn stats_of_leaf_times(leaf_ns: &[u64]) -> LeafStats {
let mut s = LeafStats {
count: 0,
sum_ns: 0,
sumsq_scaled: 0,
items: 0,
sumsq_per_item: 0,
};
for &nanos in leaf_ns {
let scaled = nanos >> 8;
s.count += 1;
s.sum_ns += nanos;
s.sumsq_scaled += scaled.saturating_mul(scaled);
}
s
}
#[test]
fn recording_a_leaf_adds_its_time_and_one_to_the_counters() {
let before = snapshot_leaf_stats();
record_leaf_time_ns(1_000);
record_leaf_time_ns(2_000);
record_leaf_time_ns(3_000);
let after = snapshot_leaf_stats();
let expected = stats_of_leaf_times(&[1_000, 2_000, 3_000]);
assert!(
after.count.wrapping_sub(before.count) >= expected.count,
"three leaves recorded, the count rose by {}",
after.count.wrapping_sub(before.count)
);
assert!(
after.sum_ns.wrapping_sub(before.sum_ns) >= expected.sum_ns,
"{} ns recorded, the sum rose by {}",
expected.sum_ns,
after.sum_ns.wrapping_sub(before.sum_ns)
);
assert!(
after.sumsq_scaled.wrapping_sub(before.sumsq_scaled) >= expected.sumsq_scaled,
"the scaled squares rose by {} against {} recorded",
after.sumsq_scaled.wrapping_sub(before.sumsq_scaled),
expected.sumsq_scaled
);
}
#[test]
fn leaf_cv_squared_handles_insufficient_samples() {
let s = stats_of_leaf_times(&[1_000, 1_000, 1_000]);
assert!(
leaf_cv_squared_per_mille(s).is_none(),
"fewer than 4 leaves should return None"
);
}
#[test]
fn leaf_cv_squared_zero_for_uniform_leaves() {
let s = stats_of_leaf_times(&[10_000; 8]);
let cv2 = leaf_cv_squared_per_mille(s).unwrap();
assert!(cv2 < 20, "uniform leaves should have cv^2 ~ 0; got {cv2}");
}
#[test]
fn leaf_cv_squared_high_for_spread_leaves() {
let s = stats_of_leaf_times(&[
1_000, 1_000, 1_000, 1_000, 100_000, 100_000, 100_000, 100_000,
]);
let cv2 = leaf_cv_squared_per_mille(s).unwrap();
assert!(
cv2 >= 500,
"spread leaves should have cv^2 >= 500; got {cv2}"
);
}
#[test]
fn leaves_shorter_than_the_unit_have_no_spread_rather_than_none_of_one() {
let s = stats_of_leaf_times(&[100, 200, 100, 200, 100, 200, 100, 200]);
assert_eq!(leaf_cv_squared_per_mille(s), None);
}
#[test]
fn items_cheaper_than_the_unit_have_no_per_item_spread() {
let leaf = 100u64 * 1024;
let per_item_sq = ((leaf as u128 * leaf as u128) / (1024u128 << 16)) as u64;
let s = LeafStats {
count: 8,
sum_ns: 8 * leaf,
sumsq_scaled: 8 * (leaf >> 8) * (leaf >> 8),
items: 8 * 1024,
sumsq_per_item: 8 * per_item_sq,
};
assert_eq!(observed_per_item_ns(s), Some(100));
assert_eq!(per_item_cv_squared_per_mille(s), None);
assert!(leaf_cv_squared_per_mille(s).is_some());
}
#[test]
fn a_window_too_sparse_to_read_keeps_the_current_multiplier() {
assert_eq!(multiplier_for_window(40, 9, None, 3), 3);
assert_eq!(multiplier_for_window(40, 9, Some(900), 3), 3);
}
#[test]
fn the_window_multiplier_takes_the_higher_of_its_two_signals() {
assert_eq!(multiplier_for_window(1000, 0, Some(900), 1), 4);
assert_eq!(multiplier_for_window(500, 500, Some(0), 1), 4);
assert_eq!(multiplier_for_window(1000, 0, Some(0), 1), 1);
}
#[test]
fn a_window_with_no_timed_leaves_reads_its_steal_rate_alone() {
assert_eq!(multiplier_for_window(500, 500, None, 1), 4);
assert_eq!(multiplier_for_window(1000, 0, None, 1), 1);
}
}