#![allow(clippy::missing_errors_doc)]
use core::sync::atomic::{AtomicU8, Ordering};
use crate::dispatch_profile::DispatchProfile;
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub enum WorkloadClass {
FineGrain,
PortBound,
LatencyBound,
MemoryBound,
Streaming,
}
impl WorkloadClass {
#[inline]
pub fn to_dispatch_profile(self) -> DispatchProfile {
match self {
WorkloadClass::FineGrain => DispatchProfile::PortBound,
WorkloadClass::PortBound => DispatchProfile::PortBound,
WorkloadClass::LatencyBound => DispatchProfile::LatencyBound,
WorkloadClass::MemoryBound => DispatchProfile::MemoryBound,
WorkloadClass::Streaming => DispatchProfile::Streaming,
}
}
}
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub enum LeafShape {
PortCompute,
LatencyCompute,
Streaming,
Gather,
Unknown,
}
impl LeafShape {
#[inline]
pub fn to_workload_class(self) -> Option<WorkloadClass> {
match self {
Self::PortCompute => Some(WorkloadClass::PortBound),
Self::LatencyCompute => Some(WorkloadClass::LatencyBound),
Self::Streaming => Some(WorkloadClass::Streaming),
Self::Gather => Some(WorkloadClass::MemoryBound),
Self::Unknown => None,
}
}
}
const TAG_LATENCY_BOUND: u8 = 0;
const TAG_PORT_BOUND: u8 = 1;
const TAG_MEMORY_BOUND: u8 = 2;
const TAG_UNSPECIFIED: u8 = 3;
const TAG_STREAMING: u8 = 4;
static ACTIVE_PROFILE_TAG: AtomicU8 = AtomicU8::new(TAG_PORT_BOUND);
#[unsafe(no_mangle)]
pub static __flynnel_marker_adaptive_profile: u8 = 0;
#[inline]
pub fn active_dispatch_profile() -> DispatchProfile {
match ACTIVE_PROFILE_TAG.load(Ordering::Acquire) {
TAG_LATENCY_BOUND => DispatchProfile::LatencyBound,
TAG_PORT_BOUND => DispatchProfile::PortBound,
TAG_MEMORY_BOUND => DispatchProfile::MemoryBound,
TAG_STREAMING => DispatchProfile::Streaming,
_ => DispatchProfile::Unspecified,
}
}
#[inline]
pub fn migrate_dispatch_profile(profile: DispatchProfile) {
let tag = match profile {
DispatchProfile::LatencyBound => TAG_LATENCY_BOUND,
DispatchProfile::PortBound => TAG_PORT_BOUND,
DispatchProfile::MemoryBound => TAG_MEMORY_BOUND,
DispatchProfile::Streaming => TAG_STREAMING,
DispatchProfile::Unspecified => TAG_UNSPECIFIED,
};
ACTIVE_PROFILE_TAG.store(tag, Ordering::Release);
}
#[inline]
pub fn migrate_workload_class(class: WorkloadClass) {
migrate_dispatch_profile(class.to_dispatch_profile());
}
#[inline]
pub fn active_workload_class() -> WorkloadClass {
match active_dispatch_profile() {
DispatchProfile::LatencyBound => WorkloadClass::LatencyBound,
DispatchProfile::PortBound => WorkloadClass::PortBound,
DispatchProfile::MemoryBound => WorkloadClass::MemoryBound,
DispatchProfile::Streaming => WorkloadClass::Streaming,
DispatchProfile::Unspecified => WorkloadClass::PortBound,
}
}
pub struct ClassThresholds {
pub fine_grain_ns: AtomicU64,
pub port_heavy_ns: AtomicU64,
pub memory_latency_ns: AtomicU64,
pub cv2_low_per_mille: AtomicU64,
pub cv2_high_per_mille: AtomicU64,
pub trivial_reduce_cycles: AtomicU64,
}
impl ClassThresholds {
pub const fn new_defaults() -> Self {
Self {
fine_grain_ns: AtomicU64::new(50),
port_heavy_ns: AtomicU64::new(500),
memory_latency_ns: AtomicU64::new(2000),
cv2_low_per_mille: AtomicU64::new(50),
cv2_high_per_mille: AtomicU64::new(500),
trivial_reduce_cycles: AtomicU64::new(30_000),
}
}
}
static CLASS_THRESHOLDS: ClassThresholds = ClassThresholds::new_defaults();
use core::sync::atomic::AtomicU64;
#[inline]
pub fn class_thresholds() -> &'static ClassThresholds {
&CLASS_THRESHOLDS
}
#[derive(Debug, Clone, Copy)]
pub struct ThresholdCalibration {
pub join_ns: u64,
pub fine_grain_ns: u64,
pub trivial_reduce_measured_cycles: u64,
pub trivial_reduce_cycles: u64,
pub smt_ratio_per_mille: u64,
pub memory_latency_ns: u64,
}
pub fn calibrate_class_thresholds() -> ThresholdCalibration {
calibrate_class_thresholds_into(class_thresholds())
}
pub fn calibrate_class_thresholds_into(target: &ClassThresholds) -> ThresholdCalibration {
use crate::sched::plan::JobPlan;
let plan = JobPlan::bare(6, 2);
for _ in 0..16 {
crate::sched::arena::join(&plan, || (), || ());
}
let mut samples = [0u64; 64];
for s in samples.iter_mut() {
let t0 = std::time::Instant::now();
crate::sched::arena::join(&plan, || (), || ());
*s = t0.elapsed().as_nanos() as u64;
}
samples.sort_unstable();
let join_ns = samples[samples.len() / 2].max(1);
let fine_grain_ns = (join_ns / 256).clamp(25, 200);
target.fine_grain_ns.store(fine_grain_ns, Ordering::Release);
let mut a = [1u64; 256];
let b = [2u64; 256];
let mut best_cycles = u64::MAX;
for _ in 0..16 {
let t0 = read_tsc_cal();
for i in 0..256 {
a[i] = a[i].wrapping_add(b[i]);
}
std::hint::black_box(&a);
let dt = read_tsc_cal().wrapping_sub(t0);
best_cycles = best_cycles.min(dt.max(1));
}
let trivial_reduce_cycles = (best_cycles.saturating_mul(8)).clamp(5_000, 200_000);
target
.trivial_reduce_cycles
.store(trivial_reduce_cycles, Ordering::Release);
let sqrt_probe = |use_smt: bool| -> u64 {
let arena = crate::sched::arena::global_local_arena();
let _guards = if use_smt { arena.acquire_smt() } else { Vec::new() };
let mut items: Vec<f64> = (1..=2048u32).map(f64::from).collect();
let probe_plan = JobPlan::bare(6, items.len() as u32);
let t0 = std::time::Instant::now();
crate::sched::par_iter::for_each_chunk(&probe_plan, &mut items, |chunk| {
for x in chunk.iter_mut() {
let mut v = *x;
for _ in 0..64 {
v = v.sqrt() + 1.0;
}
*x = v;
}
});
std::hint::black_box(&items);
t0.elapsed().as_nanos() as u64
};
let off_ns = sqrt_probe(false).max(1);
let on_ns = sqrt_probe(true).max(1);
let smt_ratio_per_mille = on_ns.saturating_mul(1000) / off_ns;
let current = target.memory_latency_ns.load(Ordering::Relaxed);
let nudged = if smt_ratio_per_mille <= 950 {
current.saturating_sub(250)
} else if smt_ratio_per_mille >= 1050 {
current.saturating_add(250)
} else {
current
};
let memory_latency_ns = nudged.clamp(1_000, 4_000);
target
.memory_latency_ns
.store(memory_latency_ns, Ordering::Release);
ThresholdCalibration {
join_ns,
fine_grain_ns,
trivial_reduce_measured_cycles: best_cycles,
trivial_reduce_cycles,
smt_ratio_per_mille,
memory_latency_ns,
}
}
#[inline]
fn read_tsc_cal() -> u64 {
#[cfg(target_arch = "x86_64")]
unsafe {
std::arch::x86_64::_rdtsc()
}
#[cfg(not(target_arch = "x86_64"))]
{
std::time::Instant::now().elapsed().as_nanos() as u64
}
}
pub fn spawn_class_threshold_calibration() {
if let Some(pool) = crate::sched::io_pool::global_io_pool() {
pool.submit(|| {
let report = calibrate_class_thresholds();
std::hint::black_box(report);
});
}
}
#[inline]
pub fn classify_observed(mean_ns: u64, cv2_per_mille: u64) -> WorkloadClass {
let t = class_thresholds();
if mean_ns < t.fine_grain_ns.load(Ordering::Relaxed) {
return WorkloadClass::FineGrain;
}
if mean_ns < t.port_heavy_ns.load(Ordering::Relaxed) {
return WorkloadClass::PortBound;
}
if cv2_per_mille < t.cv2_low_per_mille.load(Ordering::Relaxed) {
WorkloadClass::Streaming
} else if cv2_per_mille < t.cv2_high_per_mille.load(Ordering::Relaxed) {
WorkloadClass::MemoryBound
} else {
WorkloadClass::LatencyBound
}
}
#[inline]
pub fn infer_class_static(
k_outer: u8,
batch_size: u32,
estimated_per_item_ns: Option<u32>,
) -> WorkloadClass {
infer_class_static_with_shape(
k_outer,
batch_size,
estimated_per_item_ns,
LeafShape::Unknown,
)
}
#[inline]
pub fn infer_class_static_with_shape(
k_outer: u8,
batch_size: u32,
estimated_per_item_ns: Option<u32>,
shape: LeafShape,
) -> WorkloadClass {
if let Some(class) = shape.to_workload_class() {
if let Some(ns) = estimated_per_item_ns {
let total_ns = (ns as u64).saturating_mul(batch_size as u64);
if total_ns < 50_000 {
return WorkloadClass::FineGrain;
}
}
return class;
}
if let Some(ns) = estimated_per_item_ns {
let total_ns = (ns as u64).saturating_mul(batch_size as u64);
if total_ns < 50_000 {
return WorkloadClass::FineGrain;
}
let t = class_thresholds();
if (ns as u64) >= t.memory_latency_ns.load(Ordering::Relaxed) {
return WorkloadClass::LatencyBound;
}
if (ns as u64) >= t.port_heavy_ns.load(Ordering::Relaxed) {
return WorkloadClass::MemoryBound;
}
if (ns as u64) < t.fine_grain_ns.load(Ordering::Relaxed) && batch_size >= 65536 {
return WorkloadClass::Streaming;
}
return WorkloadClass::PortBound;
}
if k_outer >= 7 {
return WorkloadClass::LatencyBound;
}
if k_outer <= 3 && batch_size >= 16384 {
return WorkloadClass::Streaming;
}
if batch_size <= 32 {
return WorkloadClass::LatencyBound;
}
WorkloadClass::PortBound
}
pub const AUTO_MIGRATION_HYSTERESIS: u32 = 2;
#[inline]
fn bucket_index(class: WorkloadClass) -> u8 {
match class {
WorkloadClass::FineGrain => 0,
WorkloadClass::PortBound => 1,
WorkloadClass::Streaming => 2,
WorkloadClass::MemoryBound => 2,
WorkloadClass::LatencyBound => 3,
}
}
#[inline]
fn bucket_distance(a: WorkloadClass, b: WorkloadClass) -> u8 {
bucket_index(a).abs_diff(bucket_index(b))
}
const TAG_FINE_GRAIN: u8 = 5;
#[inline]
fn workload_class_to_tag(class: WorkloadClass) -> u8 {
match class {
WorkloadClass::FineGrain => TAG_FINE_GRAIN,
WorkloadClass::PortBound => TAG_PORT_BOUND,
WorkloadClass::LatencyBound => TAG_LATENCY_BOUND,
WorkloadClass::MemoryBound => TAG_MEMORY_BOUND,
WorkloadClass::Streaming => TAG_STREAMING,
}
}
#[inline]
pub(crate) fn class_tag_encode(class: WorkloadClass) -> u8 {
workload_class_to_tag(class)
}
#[inline]
pub(crate) fn class_tag_decode(tag: u8) -> Option<WorkloadClass> {
match tag {
TAG_LATENCY_BOUND => Some(WorkloadClass::LatencyBound),
TAG_PORT_BOUND => Some(WorkloadClass::PortBound),
TAG_MEMORY_BOUND => Some(WorkloadClass::MemoryBound),
TAG_STREAMING => Some(WorkloadClass::Streaming),
TAG_FINE_GRAIN => Some(WorkloadClass::FineGrain),
_ => None,
}
}
#[inline]
pub(crate) fn class_bucket_distance(a: WorkloadClass, b: WorkloadClass) -> u8 {
bucket_distance(a, b)
}
static AUTO_PENDING_TAG: AtomicU8 = AtomicU8::new(TAG_PORT_BOUND);
static AUTO_PENDING_RUN: core::sync::atomic::AtomicU32 =
core::sync::atomic::AtomicU32::new(0);
static AUTO_LAST_COUNT: core::sync::atomic::AtomicU64 =
core::sync::atomic::AtomicU64::new(0);
static AUTO_LAST_SUM_NS: core::sync::atomic::AtomicU64 =
core::sync::atomic::AtomicU64::new(0);
static AUTO_LAST_SUMSQ: core::sync::atomic::AtomicU64 =
core::sync::atomic::AtomicU64::new(0);
#[inline]
pub fn tick_auto_classify() {
use crate::sched::split_observer::snapshot_leaf_stats;
let stats = snapshot_leaf_stats();
let prev_count = AUTO_LAST_COUNT.load(Ordering::Relaxed);
let prev_sum = AUTO_LAST_SUM_NS.load(Ordering::Relaxed);
let prev_sumsq = AUTO_LAST_SUMSQ.load(Ordering::Relaxed);
let dcount = stats.count.saturating_sub(prev_count);
if dcount < 4 {
return;
}
let dsum = stats.sum_ns.saturating_sub(prev_sum);
let dsumsq = stats.sumsq_scaled.saturating_sub(prev_sumsq);
AUTO_LAST_COUNT.store(stats.count, Ordering::Relaxed);
AUTO_LAST_SUM_NS.store(stats.sum_ns, Ordering::Relaxed);
AUTO_LAST_SUMSQ.store(stats.sumsq_scaled, Ordering::Relaxed);
let mean_ns = dsum / dcount;
let scaled_mean = (dsum >> 8) / dcount;
let cv2 = if scaled_mean == 0 {
0
} else {
let sumsq_per_n = dsumsq / dcount;
let mean_sq = scaled_mean.saturating_mul(scaled_mean);
let var = sumsq_per_n.saturating_sub(mean_sq);
var.saturating_mul(1000) / mean_sq.max(1)
};
let observed = classify_observed(mean_ns, cv2);
let observed_tag = workload_class_to_tag(observed);
let active = active_workload_class();
if active != observed
&& bucket_distance(active, observed) >= 2
&& dcount >= 64
{
migrate_workload_class(observed);
AUTO_PENDING_TAG.store(observed_tag, Ordering::Relaxed);
AUTO_PENDING_RUN.store(0, Ordering::Relaxed);
return;
}
let pending_tag = AUTO_PENDING_TAG.load(Ordering::Relaxed);
if pending_tag == observed_tag {
let prior_run = AUTO_PENDING_RUN.fetch_add(1, Ordering::Relaxed);
let new_run = prior_run.saturating_add(1);
if new_run >= AUTO_MIGRATION_HYSTERESIS && active != observed {
migrate_workload_class(observed);
}
} else {
AUTO_PENDING_TAG.store(observed_tag, Ordering::Relaxed);
AUTO_PENDING_RUN.store(1, Ordering::Relaxed);
}
}
pub fn reset_auto_classify_state() {
AUTO_PENDING_TAG.store(TAG_PORT_BOUND, Ordering::Relaxed);
AUTO_PENDING_RUN.store(0, Ordering::Relaxed);
AUTO_LAST_COUNT.store(0, Ordering::Relaxed);
AUTO_LAST_SUM_NS.store(0, Ordering::Relaxed);
AUTO_LAST_SUMSQ.store(0, Ordering::Relaxed);
}
#[cfg(test)]
pub(crate) fn global_profile_test_lock() -> std::sync::MutexGuard<'static, ()> {
static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
LOCK.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn threshold_calibration_installs_clamped_values() {
let local = ClassThresholds::new_defaults();
let report = calibrate_class_thresholds_into(&local);
assert!(
(25..=200).contains(&report.fine_grain_ns),
"fine_grain_ns {} outside clamp band",
report.fine_grain_ns
);
assert!(
(5_000..=200_000).contains(&report.trivial_reduce_cycles),
"trivial_reduce_cycles {} outside clamp band",
report.trivial_reduce_cycles
);
assert!(
(1_000..=4_000).contains(&report.memory_latency_ns),
"memory_latency_ns {} outside clamp band",
report.memory_latency_ns
);
assert_eq!(
local.fine_grain_ns.load(Ordering::Relaxed),
report.fine_grain_ns
);
assert_eq!(
local.trivial_reduce_cycles.load(Ordering::Relaxed),
report.trivial_reduce_cycles
);
assert_eq!(
local.memory_latency_ns.load(Ordering::Relaxed),
report.memory_latency_ns
);
}
fn restore_default_profile() {
migrate_dispatch_profile(DispatchProfile::PortBound);
}
#[test]
fn workload_class_maps_to_dispatch_profile() {
assert_eq!(WorkloadClass::FineGrain.to_dispatch_profile(), DispatchProfile::PortBound);
assert_eq!(WorkloadClass::PortBound.to_dispatch_profile(), DispatchProfile::PortBound);
assert_eq!(WorkloadClass::LatencyBound.to_dispatch_profile(), DispatchProfile::LatencyBound);
assert_eq!(WorkloadClass::MemoryBound.to_dispatch_profile(), DispatchProfile::MemoryBound);
assert_eq!(WorkloadClass::Streaming.to_dispatch_profile(), DispatchProfile::Streaming);
}
#[test]
fn migration_changes_active_profile() {
let _guard = TestGuard::new();
let initial = active_dispatch_profile();
assert!(matches!(initial, DispatchProfile::PortBound),
"default expected PortBound, got {initial:?}");
migrate_dispatch_profile(DispatchProfile::LatencyBound);
assert_eq!(active_dispatch_profile(), DispatchProfile::LatencyBound);
migrate_dispatch_profile(DispatchProfile::MemoryBound);
assert_eq!(active_dispatch_profile(), DispatchProfile::MemoryBound);
migrate_dispatch_profile(DispatchProfile::Streaming);
assert_eq!(active_dispatch_profile(), DispatchProfile::Streaming);
migrate_dispatch_profile(DispatchProfile::PortBound);
assert_eq!(active_dispatch_profile(), DispatchProfile::PortBound);
}
#[test]
fn classifier_picks_fine_grain_for_tiny_leaves() {
assert_eq!(classify_observed(10, 0), WorkloadClass::FineGrain);
assert_eq!(classify_observed(49, 1000), WorkloadClass::FineGrain);
}
#[test]
fn classifier_picks_port_bound_for_medium_leaves() {
assert_eq!(classify_observed(100, 10), WorkloadClass::PortBound);
assert_eq!(classify_observed(499, 5000), WorkloadClass::PortBound);
}
#[test]
fn classifier_picks_streaming_for_heavy_uniform_leaves() {
assert_eq!(classify_observed(500, 0), WorkloadClass::Streaming);
assert_eq!(classify_observed(10_000, 49), WorkloadClass::Streaming);
}
#[test]
fn classifier_picks_memory_bound_for_heavy_moderate_variance() {
assert_eq!(classify_observed(500, 100), WorkloadClass::MemoryBound);
assert_eq!(classify_observed(2000, 499), WorkloadClass::MemoryBound);
}
#[test]
fn classifier_picks_latency_bound_for_heavy_high_variance() {
assert_eq!(classify_observed(500, 500), WorkloadClass::LatencyBound);
assert_eq!(classify_observed(5000, 10_000), WorkloadClass::LatencyBound);
}
#[test]
fn static_classifier_picks_streaming_for_big_byte_scan() {
assert_eq!(
infer_class_static(6, 16 * 1024 * 1024, Some(1)),
WorkloadClass::Streaming,
);
}
#[test]
fn static_classifier_picks_streaming_for_big_image() {
assert_eq!(
infer_class_static(6, 2048 * 2048, Some(10)),
WorkloadClass::Streaming,
);
}
#[test]
fn static_classifier_picks_port_bound_for_medium_per_item() {
assert_eq!(
infer_class_static(6, 100 * 1024, Some(100)),
WorkloadClass::PortBound,
);
}
#[test]
fn static_classifier_picks_memory_bound_for_heavy_per_item() {
assert_eq!(
infer_class_static(6, 1024, Some(600)),
WorkloadClass::MemoryBound,
);
}
#[test]
fn static_classifier_picks_fine_grain_for_tiny_total() {
assert_eq!(
infer_class_static(6, 100, Some(10)),
WorkloadClass::FineGrain,
);
}
#[test]
fn static_classifier_picks_latency_for_high_precision_tier() {
assert_eq!(
infer_class_static(12, 1024, None),
WorkloadClass::LatencyBound,
);
}
#[test]
fn static_classifier_picks_streaming_for_low_tier_large_batch() {
assert_eq!(
infer_class_static(2, 100_000, None),
WorkloadClass::Streaming,
);
}
#[test]
fn static_classifier_falls_back_to_port_bound_when_unsure() {
assert_eq!(
infer_class_static(6, 1024, None),
WorkloadClass::PortBound,
);
}
#[test]
fn auto_classify_migrates_after_hysteresis() {
use crate::sched::split_observer::{
acquire_test_lock, record_leaf_batch, reset_leaf_stats,
};
let _stats_lock = acquire_test_lock();
let _guard = TestGuard::new();
reset_leaf_stats();
reset_auto_classify_state();
migrate_dispatch_profile(DispatchProfile::PortBound);
assert_eq!(active_workload_class(), WorkloadClass::PortBound);
let sample_ns: u64 = 1000;
let count: u64 = 64;
let scaled = sample_ns >> 8;
let sumsq = scaled.saturating_mul(scaled).saturating_mul(count);
for _ in 0..AUTO_MIGRATION_HYSTERESIS {
record_leaf_batch(sample_ns * count, sumsq, count);
}
assert_eq!(
active_workload_class(),
WorkloadClass::Streaming,
"auto-classifier did not migrate after {} consecutive Streaming observations",
AUTO_MIGRATION_HYSTERESIS,
);
}
#[test]
fn auto_classify_does_not_migrate_on_a_single_outlier() {
use crate::sched::split_observer::{
acquire_test_lock, record_leaf_batch, reset_leaf_stats,
};
let _stats_lock = acquire_test_lock();
let _guard = TestGuard::new();
reset_leaf_stats();
reset_auto_classify_state();
migrate_dispatch_profile(DispatchProfile::PortBound);
let sample_ns: u64 = 1000;
let count: u64 = 64;
let scaled = sample_ns >> 8;
let sumsq = scaled.saturating_mul(scaled).saturating_mul(count);
record_leaf_batch(sample_ns * count, sumsq, count);
assert_eq!(
active_workload_class(),
WorkloadClass::PortBound,
"auto-classifier migrated on a single observation; hysteresis broken",
);
}
#[test]
fn workload_class_migration_propagates() {
let _guard = TestGuard::new();
migrate_workload_class(WorkloadClass::LatencyBound);
assert_eq!(active_dispatch_profile(), DispatchProfile::LatencyBound);
assert_eq!(active_workload_class(), WorkloadClass::LatencyBound);
migrate_workload_class(WorkloadClass::MemoryBound);
assert_eq!(active_dispatch_profile(), DispatchProfile::MemoryBound);
assert_eq!(active_workload_class(), WorkloadClass::MemoryBound);
}
struct TestGuard {
lock: std::sync::MutexGuard<'static, ()>,
}
impl TestGuard {
fn new() -> Self {
let lock = super::global_profile_test_lock();
restore_default_profile();
Self { lock }
}
}
impl Drop for TestGuard {
fn drop(&mut self) {
restore_default_profile();
let _still_held = &self.lock;
}
}
}