#![allow(clippy::missing_errors_doc)]
use core::sync::atomic::{AtomicU8, Ordering};
use crate::dispatch_profile::DispatchProfile;
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub enum WorkloadClass {
FineGrain,
PortBound,
LatencyBound,
MemoryBound,
Streaming,
}
impl WorkloadClass {
#[inline]
pub fn to_dispatch_profile(self) -> DispatchProfile {
match self {
WorkloadClass::FineGrain => DispatchProfile::PortBound,
WorkloadClass::PortBound => DispatchProfile::PortBound,
WorkloadClass::LatencyBound => DispatchProfile::LatencyBound,
WorkloadClass::MemoryBound => DispatchProfile::MemoryBound,
WorkloadClass::Streaming => DispatchProfile::Streaming,
}
}
}
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub enum LeafShape {
PortCompute,
LatencyCompute,
Streaming,
Gather,
Unknown,
}
impl LeafShape {
#[inline]
pub fn to_workload_class(self) -> Option<WorkloadClass> {
match self {
Self::PortCompute => Some(WorkloadClass::PortBound),
Self::LatencyCompute => Some(WorkloadClass::LatencyBound),
Self::Streaming => Some(WorkloadClass::Streaming),
Self::Gather => Some(WorkloadClass::MemoryBound),
Self::Unknown => None,
}
}
}
const TAG_LATENCY_BOUND: u8 = 0;
const TAG_PORT_BOUND: u8 = 1;
const TAG_MEMORY_BOUND: u8 = 2;
const TAG_UNSPECIFIED: u8 = 3;
const TAG_STREAMING: u8 = 4;
static ACTIVE_PROFILE_TAG: AtomicU8 = AtomicU8::new(ACTIVE_PROFILE_TAG_DEFAULT);
const ACTIVE_PROFILE_TAG_DEFAULT: u8 = TAG_PORT_BOUND;
#[unsafe(no_mangle)]
pub static __flynnel_marker_adaptive_profile: u8 = 0;
#[inline]
const fn profile_of_tag(tag: u8) -> DispatchProfile {
match tag {
TAG_LATENCY_BOUND => DispatchProfile::LatencyBound,
TAG_PORT_BOUND => DispatchProfile::PortBound,
TAG_MEMORY_BOUND => DispatchProfile::MemoryBound,
TAG_STREAMING => DispatchProfile::Streaming,
_ => DispatchProfile::Unspecified,
}
}
#[inline]
const fn tag_of_profile(profile: DispatchProfile) -> u8 {
match profile {
DispatchProfile::LatencyBound => TAG_LATENCY_BOUND,
DispatchProfile::PortBound => TAG_PORT_BOUND,
DispatchProfile::MemoryBound => TAG_MEMORY_BOUND,
DispatchProfile::Streaming => TAG_STREAMING,
DispatchProfile::Unspecified => TAG_UNSPECIFIED,
}
}
#[inline]
pub fn active_dispatch_profile() -> DispatchProfile {
profile_of_tag(ACTIVE_PROFILE_TAG.load(Ordering::Acquire))
}
#[inline]
pub fn migrate_dispatch_profile(profile: DispatchProfile) {
ACTIVE_PROFILE_TAG.store(tag_of_profile(profile), Ordering::Release);
}
#[inline]
pub fn migrate_workload_class(class: WorkloadClass) {
migrate_dispatch_profile(class.to_dispatch_profile());
}
#[inline]
pub fn active_workload_class() -> WorkloadClass {
class_of_profile(active_dispatch_profile())
}
#[inline]
const fn class_of_profile(profile: DispatchProfile) -> WorkloadClass {
match profile {
DispatchProfile::LatencyBound => WorkloadClass::LatencyBound,
DispatchProfile::PortBound => WorkloadClass::PortBound,
DispatchProfile::MemoryBound => WorkloadClass::MemoryBound,
DispatchProfile::Streaming => WorkloadClass::Streaming,
DispatchProfile::Unspecified => WorkloadClass::PortBound,
}
}
pub struct ClassThresholds {
pub fine_grain_ns: AtomicU64,
pub port_heavy_ns: AtomicU64,
pub memory_latency_ns: AtomicU64,
pub cv2_low_per_mille: AtomicU64,
pub cv2_high_per_mille: AtomicU64,
pub trivial_reduce_cycles: AtomicU64,
}
impl ClassThresholds {
pub const fn new_defaults() -> Self {
Self {
fine_grain_ns: AtomicU64::new(50),
port_heavy_ns: AtomicU64::new(500),
memory_latency_ns: AtomicU64::new(2000),
cv2_low_per_mille: AtomicU64::new(50),
cv2_high_per_mille: AtomicU64::new(500),
trivial_reduce_cycles: AtomicU64::new(30_000),
}
}
}
static CLASS_THRESHOLDS: ClassThresholds = ClassThresholds::new_defaults();
use core::sync::atomic::AtomicU64;
#[inline]
pub fn class_thresholds() -> &'static ClassThresholds {
&CLASS_THRESHOLDS
}
#[derive(Debug, Clone, Copy)]
pub struct ThresholdCalibration {
pub join_ns: u64,
pub fine_grain_ns: u64,
pub trivial_reduce_measured_cycles: u64,
pub trivial_reduce_cycles: u64,
pub smt_ratio_per_mille: u64,
pub memory_latency_ns: u64,
}
pub fn calibrate_class_thresholds() -> ThresholdCalibration {
calibrate_class_thresholds_into(class_thresholds())
}
pub fn calibrate_class_thresholds_into(target: &ClassThresholds) -> ThresholdCalibration {
use crate::sched::plan::JobPlan;
let plan = JobPlan::bare(6, 2);
for _ in 0..16 {
crate::sched::arena::join(&plan, || (), || ());
}
let mut samples = [0u64; 64];
for s in samples.iter_mut() {
let t0 = std::time::Instant::now();
crate::sched::arena::join(&plan, || (), || ());
*s = t0.elapsed().as_nanos() as u64;
}
samples.sort_unstable();
let join_ns = samples[samples.len() / 2].max(1);
let fine_grain_ns = (join_ns / 256).clamp(25, 200);
target.fine_grain_ns.store(fine_grain_ns, Ordering::Release);
let mut a = [1u64; 256];
let b = [2u64; 256];
let mut best_cycles = u64::MAX;
for _ in 0..16 {
let t0 = read_tsc_cal();
for i in 0..256 {
a[i] = a[i].wrapping_add(b[i]);
}
std::hint::black_box(&a);
let dt = read_tsc_cal().wrapping_sub(t0);
best_cycles = best_cycles.min(dt.max(1));
}
let trivial_reduce_cycles = (best_cycles.saturating_mul(8)).clamp(5_000, 200_000);
target
.trivial_reduce_cycles
.store(trivial_reduce_cycles, Ordering::Release);
let sqrt_probe = |use_smt: bool| -> u64 {
let arena = crate::sched::arena::global_local_arena();
let _guards = if use_smt { arena.acquire_smt() } else { Vec::new() };
let mut items: Vec<f64> = (1..=2048u32).map(f64::from).collect();
let probe_plan = JobPlan::bare(6, items.len() as u32);
let t0 = std::time::Instant::now();
crate::sched::par_iter::for_each_chunk(&probe_plan, &mut items, |chunk| {
for x in chunk.iter_mut() {
let mut v = *x;
for _ in 0..64 {
v = v.sqrt() + 1.0;
}
*x = v;
}
});
std::hint::black_box(&items);
t0.elapsed().as_nanos() as u64
};
let off_ns = sqrt_probe(false).max(1);
let on_ns = sqrt_probe(true).max(1);
let smt_ratio_per_mille = on_ns.saturating_mul(1000) / off_ns;
let current = target.memory_latency_ns.load(Ordering::Relaxed);
let nudged = if smt_ratio_per_mille <= 950 {
current.saturating_sub(250)
} else if smt_ratio_per_mille >= 1050 {
current.saturating_add(250)
} else {
current
};
let memory_latency_ns = nudged.clamp(1_000, 4_000);
target
.memory_latency_ns
.store(memory_latency_ns, Ordering::Release);
ThresholdCalibration {
join_ns,
fine_grain_ns,
trivial_reduce_measured_cycles: best_cycles,
trivial_reduce_cycles,
smt_ratio_per_mille,
memory_latency_ns,
}
}
#[inline]
fn read_tsc_cal() -> u64 {
#[cfg(target_arch = "x86_64")]
unsafe {
std::arch::x86_64::_rdtsc()
}
#[cfg(not(target_arch = "x86_64"))]
{
std::time::Instant::now().elapsed().as_nanos() as u64
}
}
pub fn spawn_class_threshold_calibration() {
if let Some(pool) = crate::sched::io_pool::global_io_pool() {
pool.submit(|| {
let report = calibrate_class_thresholds();
std::hint::black_box(report);
});
}
}
#[inline]
pub fn classify_observed(mean_ns: u64, cv2_per_mille: u64) -> WorkloadClass {
let t = class_thresholds();
match class_from_mean(t, mean_ns) {
Some(class) => class,
None => class_from_spread(t, cv2_per_mille),
}
}
#[inline]
fn class_from_mean(t: &ClassThresholds, mean_ns: u64) -> Option<WorkloadClass> {
if mean_ns < t.fine_grain_ns.load(Ordering::Relaxed) {
Some(WorkloadClass::FineGrain)
} else if mean_ns < t.port_heavy_ns.load(Ordering::Relaxed) {
Some(WorkloadClass::PortBound)
} else {
None
}
}
#[inline]
fn class_from_spread(t: &ClassThresholds, cv2_per_mille: u64) -> WorkloadClass {
if cv2_per_mille < t.cv2_low_per_mille.load(Ordering::Relaxed) {
WorkloadClass::Streaming
} else if cv2_per_mille < t.cv2_high_per_mille.load(Ordering::Relaxed) {
WorkloadClass::MemoryBound
} else {
WorkloadClass::LatencyBound
}
}
#[inline]
pub(crate) fn classify_window(mean_ns: u64, cv2_per_mille: Option<u64>) -> Option<WorkloadClass> {
let t = class_thresholds();
match (class_from_mean(t, mean_ns), cv2_per_mille) {
(Some(class), _) => Some(class),
(None, Some(cv2)) => Some(class_from_spread(t, cv2)),
(None, None) => None,
}
}
#[inline]
pub fn infer_class_static(
k_outer: u8,
batch_size: u32,
estimated_per_item_ns: Option<u32>,
) -> WorkloadClass {
infer_class_static_with_shape(
k_outer,
batch_size,
estimated_per_item_ns,
LeafShape::Unknown,
)
}
#[inline]
pub fn infer_class_static_with_shape(
k_outer: u8,
batch_size: u32,
estimated_per_item_ns: Option<u32>,
shape: LeafShape,
) -> WorkloadClass {
if let Some(class) = shape.to_workload_class() {
if let Some(ns) = estimated_per_item_ns {
let total_ns = (ns as u64).saturating_mul(batch_size as u64);
if total_ns < 50_000 {
return WorkloadClass::FineGrain;
}
}
return class;
}
if let Some(ns) = estimated_per_item_ns {
let total_ns = (ns as u64).saturating_mul(batch_size as u64);
if total_ns < 50_000 {
return WorkloadClass::FineGrain;
}
let t = class_thresholds();
if (ns as u64) >= t.memory_latency_ns.load(Ordering::Relaxed) {
return WorkloadClass::LatencyBound;
}
if (ns as u64) >= t.port_heavy_ns.load(Ordering::Relaxed) {
return WorkloadClass::MemoryBound;
}
if (ns as u64) < t.fine_grain_ns.load(Ordering::Relaxed) && batch_size >= 65536 {
return WorkloadClass::Streaming;
}
return WorkloadClass::PortBound;
}
if k_outer >= 7 {
return WorkloadClass::LatencyBound;
}
if k_outer <= 3 && batch_size >= 16384 {
return WorkloadClass::Streaming;
}
if batch_size <= 32 {
return WorkloadClass::LatencyBound;
}
WorkloadClass::PortBound
}
pub const AUTO_MIGRATION_HYSTERESIS: u32 = 2;
#[inline]
fn bucket_index(class: WorkloadClass) -> u8 {
match class {
WorkloadClass::FineGrain => 0,
WorkloadClass::PortBound => 1,
WorkloadClass::Streaming => 2,
WorkloadClass::MemoryBound => 2,
WorkloadClass::LatencyBound => 3,
}
}
#[inline]
fn bucket_distance(a: WorkloadClass, b: WorkloadClass) -> u8 {
bucket_index(a).abs_diff(bucket_index(b))
}
const TAG_FINE_GRAIN: u8 = 5;
#[inline]
fn workload_class_to_tag(class: WorkloadClass) -> u8 {
match class {
WorkloadClass::FineGrain => TAG_FINE_GRAIN,
WorkloadClass::PortBound => TAG_PORT_BOUND,
WorkloadClass::LatencyBound => TAG_LATENCY_BOUND,
WorkloadClass::MemoryBound => TAG_MEMORY_BOUND,
WorkloadClass::Streaming => TAG_STREAMING,
}
}
#[inline]
pub(crate) fn class_tag_encode(class: WorkloadClass) -> u8 {
workload_class_to_tag(class)
}
#[inline]
pub(crate) fn class_tag_decode(tag: u8) -> Option<WorkloadClass> {
match tag {
TAG_LATENCY_BOUND => Some(WorkloadClass::LatencyBound),
TAG_PORT_BOUND => Some(WorkloadClass::PortBound),
TAG_MEMORY_BOUND => Some(WorkloadClass::MemoryBound),
TAG_STREAMING => Some(WorkloadClass::Streaming),
TAG_FINE_GRAIN => Some(WorkloadClass::FineGrain),
_ => None,
}
}
#[inline]
pub(crate) fn class_bucket_distance(a: WorkloadClass, b: WorkloadClass) -> u8 {
bucket_distance(a, b)
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
struct PendingRun {
tag: u8,
run: u32,
}
impl PendingRun {
const fn fresh() -> Self {
Self {
tag: TAG_PORT_BOUND,
run: 0,
}
}
const fn packed(self) -> u64 {
((self.tag as u64) << 32) | self.run as u64
}
const fn unpacked(word: u64) -> Self {
Self {
tag: (word >> 32) as u8,
run: word as u32,
}
}
fn observe(
self,
active: WorkloadClass,
observed: WorkloadClass,
dcount: u64,
) -> (Option<WorkloadClass>, Self) {
let tag = workload_class_to_tag(observed);
if active != observed && bucket_distance(active, observed) >= 2 && dcount >= 64 {
return (Some(observed), Self { tag, run: 0 });
}
if self.tag != tag {
return (None, Self { tag, run: 1 });
}
let run = self.run.saturating_add(1);
let migrate = run >= AUTO_MIGRATION_HYSTERESIS && active != observed;
(migrate.then_some(observed), Self { tag, run })
}
}
static AUTO_PENDING: core::sync::atomic::AtomicU64 =
core::sync::atomic::AtomicU64::new(PendingRun::fresh().packed());
static AUTO_LAST_COUNT: core::sync::atomic::AtomicU64 =
core::sync::atomic::AtomicU64::new(0);
static AUTO_LAST_SUM_NS: core::sync::atomic::AtomicU64 =
core::sync::atomic::AtomicU64::new(0);
static AUTO_LAST_SUMSQ: core::sync::atomic::AtomicU64 =
core::sync::atomic::AtomicU64::new(0);
static AUTO_LAST_ITEMS: core::sync::atomic::AtomicU64 =
core::sync::atomic::AtomicU64::new(0);
static AUTO_LAST_SUMSQ_PER_ITEM: core::sync::atomic::AtomicU64 =
core::sync::atomic::AtomicU64::new(0);
#[inline]
pub fn tick_auto_classify() {
use crate::sched::split_observer::snapshot_leaf_stats;
let stats = snapshot_leaf_stats();
let prev_count = AUTO_LAST_COUNT.load(Ordering::Relaxed);
let prev_sum = AUTO_LAST_SUM_NS.load(Ordering::Relaxed);
let prev_sumsq = AUTO_LAST_SUMSQ.load(Ordering::Relaxed);
let dcount = stats.count.saturating_sub(prev_count);
if dcount < 4 {
return;
}
let dsum = stats.sum_ns.saturating_sub(prev_sum);
let dsumsq = stats.sumsq_scaled.saturating_sub(prev_sumsq);
let ditems = stats
.items
.saturating_sub(AUTO_LAST_ITEMS.load(Ordering::Relaxed));
let dsumsq_per_item = stats
.sumsq_per_item
.saturating_sub(AUTO_LAST_SUMSQ_PER_ITEM.load(Ordering::Relaxed));
AUTO_LAST_COUNT.store(stats.count, Ordering::Relaxed);
AUTO_LAST_SUM_NS.store(stats.sum_ns, Ordering::Relaxed);
AUTO_LAST_SUMSQ.store(stats.sumsq_scaled, Ordering::Relaxed);
AUTO_LAST_ITEMS.store(stats.items, Ordering::Relaxed);
AUTO_LAST_SUMSQ_PER_ITEM.store(stats.sumsq_per_item, Ordering::Relaxed);
let (mean_ns, cv2) = window_reading(dcount, dsum, dsumsq, ditems, dsumsq_per_item);
let Some(observed) = classify_window(mean_ns, cv2) else {
return;
};
let active = active_workload_class();
let mut word = AUTO_PENDING.load(Ordering::Relaxed);
loop {
let (migrate, next) = PendingRun::unpacked(word).observe(active, observed, dcount);
match AUTO_PENDING.compare_exchange_weak(
word,
next.packed(),
Ordering::Relaxed,
Ordering::Relaxed,
) {
Ok(_) => {
if let Some(class) = migrate {
migrate_workload_class(class);
}
return;
}
Err(current) => word = current,
}
}
}
fn window_reading(
dcount: u64,
dsum: u64,
dsumsq: u64,
ditems: u64,
dsumsq_per_item: u64,
) -> (u64, Option<u64>) {
if let Some(mean) = dsum.checked_div(ditems) {
let mean_sq = ((mean as u128).saturating_mul(mean as u128) >> 16) as u64;
let spread = (mean_sq > 0).then(|| {
let var = dsumsq_per_item.saturating_sub(mean_sq.saturating_mul(ditems)) / ditems;
var.saturating_mul(1000) / mean_sq
});
(mean, spread)
} else {
let mean = dsum / dcount;
let scaled_mean = (dsum >> 8) / dcount;
let spread = (scaled_mean > 0).then(|| {
let sumsq_per_n = dsumsq / dcount;
let mean_sq = scaled_mean.saturating_mul(scaled_mean);
let var = sumsq_per_n.saturating_sub(mean_sq);
var.saturating_mul(1000) / mean_sq
});
(mean, spread)
}
}
pub fn reset_auto_classify_state() {
AUTO_PENDING.store(PendingRun::fresh().packed(), Ordering::Relaxed);
AUTO_LAST_COUNT.store(0, Ordering::Relaxed);
AUTO_LAST_SUM_NS.store(0, Ordering::Relaxed);
AUTO_LAST_SUMSQ.store(0, Ordering::Relaxed);
AUTO_LAST_ITEMS.store(0, Ordering::Relaxed);
AUTO_LAST_SUMSQ_PER_ITEM.store(0, Ordering::Relaxed);
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn threshold_calibration_installs_clamped_values() {
let local = ClassThresholds::new_defaults();
let report = calibrate_class_thresholds_into(&local);
assert!(
(25..=200).contains(&report.fine_grain_ns),
"fine_grain_ns {} outside clamp band",
report.fine_grain_ns
);
assert!(
(5_000..=200_000).contains(&report.trivial_reduce_cycles),
"trivial_reduce_cycles {} outside clamp band",
report.trivial_reduce_cycles
);
assert!(
(1_000..=4_000).contains(&report.memory_latency_ns),
"memory_latency_ns {} outside clamp band",
report.memory_latency_ns
);
assert_eq!(
local.fine_grain_ns.load(Ordering::Relaxed),
report.fine_grain_ns
);
assert_eq!(
local.trivial_reduce_cycles.load(Ordering::Relaxed),
report.trivial_reduce_cycles
);
assert_eq!(
local.memory_latency_ns.load(Ordering::Relaxed),
report.memory_latency_ns
);
}
#[test]
fn workload_class_maps_to_dispatch_profile() {
assert_eq!(WorkloadClass::FineGrain.to_dispatch_profile(), DispatchProfile::PortBound);
assert_eq!(WorkloadClass::PortBound.to_dispatch_profile(), DispatchProfile::PortBound);
assert_eq!(WorkloadClass::LatencyBound.to_dispatch_profile(), DispatchProfile::LatencyBound);
assert_eq!(WorkloadClass::MemoryBound.to_dispatch_profile(), DispatchProfile::MemoryBound);
assert_eq!(WorkloadClass::Streaming.to_dispatch_profile(), DispatchProfile::Streaming);
}
#[test]
fn every_profile_survives_its_tag_and_the_cell_starts_port_bound() {
for profile in [
DispatchProfile::LatencyBound,
DispatchProfile::PortBound,
DispatchProfile::MemoryBound,
DispatchProfile::Streaming,
DispatchProfile::Unspecified,
] {
assert_eq!(profile_of_tag(tag_of_profile(profile)), profile);
}
assert_eq!(profile_of_tag(TAG_PORT_BOUND), DispatchProfile::PortBound);
assert_eq!(
profile_of_tag(ACTIVE_PROFILE_TAG_DEFAULT),
DispatchProfile::PortBound,
"the cell's initial value is the calibration default"
);
assert_eq!(
profile_of_tag(200),
DispatchProfile::Unspecified,
"a tag no profile is stored as reads as unspecified"
);
}
#[test]
fn classifier_picks_fine_grain_for_tiny_leaves() {
assert_eq!(classify_observed(10, 0), WorkloadClass::FineGrain);
assert_eq!(classify_observed(49, 1000), WorkloadClass::FineGrain);
}
#[test]
fn classifier_picks_port_bound_for_medium_leaves() {
assert_eq!(classify_observed(100, 10), WorkloadClass::PortBound);
assert_eq!(classify_observed(499, 5000), WorkloadClass::PortBound);
}
#[test]
fn classifier_picks_streaming_for_heavy_uniform_leaves() {
assert_eq!(classify_observed(500, 0), WorkloadClass::Streaming);
assert_eq!(classify_observed(10_000, 49), WorkloadClass::Streaming);
}
#[test]
fn classifier_picks_memory_bound_for_heavy_moderate_variance() {
assert_eq!(classify_observed(500, 100), WorkloadClass::MemoryBound);
assert_eq!(classify_observed(2000, 499), WorkloadClass::MemoryBound);
}
#[test]
fn classifier_picks_latency_bound_for_heavy_high_variance() {
assert_eq!(classify_observed(500, 500), WorkloadClass::LatencyBound);
assert_eq!(classify_observed(5000, 10_000), WorkloadClass::LatencyBound);
}
#[test]
fn static_classifier_picks_streaming_for_big_byte_scan() {
assert_eq!(
infer_class_static(6, 16 * 1024 * 1024, Some(1)),
WorkloadClass::Streaming,
);
}
#[test]
fn static_classifier_picks_streaming_for_big_image() {
assert_eq!(
infer_class_static(6, 2048 * 2048, Some(10)),
WorkloadClass::Streaming,
);
}
#[test]
fn static_classifier_picks_port_bound_for_medium_per_item() {
assert_eq!(
infer_class_static(6, 100 * 1024, Some(100)),
WorkloadClass::PortBound,
);
}
#[test]
fn static_classifier_picks_memory_bound_for_heavy_per_item() {
assert_eq!(
infer_class_static(6, 1024, Some(600)),
WorkloadClass::MemoryBound,
);
}
#[test]
fn static_classifier_picks_fine_grain_for_tiny_total() {
assert_eq!(
infer_class_static(6, 100, Some(10)),
WorkloadClass::FineGrain,
);
}
#[test]
fn static_classifier_picks_latency_for_high_precision_tier() {
assert_eq!(
infer_class_static(12, 1024, None),
WorkloadClass::LatencyBound,
);
}
#[test]
fn static_classifier_picks_streaming_for_low_tier_large_batch() {
assert_eq!(
infer_class_static(2, 100_000, None),
WorkloadClass::Streaming,
);
}
#[test]
fn static_classifier_falls_back_to_port_bound_when_unsure() {
assert_eq!(
infer_class_static(6, 1024, None),
WorkloadClass::PortBound,
);
}
#[test]
fn auto_classify_migrates_after_hysteresis() {
let mut state = PendingRun::fresh();
for seen in 1..AUTO_MIGRATION_HYSTERESIS {
let (migrate, next) =
state.observe(WorkloadClass::PortBound, WorkloadClass::Streaming, 16);
assert_eq!(
migrate, None,
"observation {seen} of {AUTO_MIGRATION_HYSTERESIS} migrated early"
);
state = next;
}
let (migrate, next) = state.observe(WorkloadClass::PortBound, WorkloadClass::Streaming, 16);
assert_eq!(
migrate,
Some(WorkloadClass::Streaming),
"auto-classifier did not migrate after {AUTO_MIGRATION_HYSTERESIS} consecutive \
Streaming observations"
);
assert_eq!(next.run, AUTO_MIGRATION_HYSTERESIS);
}
#[test]
fn auto_classify_does_not_migrate_on_a_single_outlier() {
let (migrate, next) =
PendingRun::fresh().observe(WorkloadClass::PortBound, WorkloadClass::Streaming, 16);
assert_eq!(
migrate, None,
"auto-classifier migrated on a single observation; hysteresis broken"
);
assert_eq!(
next,
PendingRun {
tag: workload_class_to_tag(WorkloadClass::Streaming),
run: 1,
}
);
let (migrate, next) =
next.observe(WorkloadClass::PortBound, WorkloadClass::MemoryBound, 16);
assert_eq!(migrate, None);
assert_eq!(next.run, 1);
}
#[test]
fn auto_classify_migrates_at_once_when_the_observation_is_far_and_wide() {
let (migrate, next) =
PendingRun::fresh().observe(WorkloadClass::PortBound, WorkloadClass::LatencyBound, 64);
assert_eq!(migrate, Some(WorkloadClass::LatencyBound));
assert_eq!(next.run, 0);
let (migrate, next) =
PendingRun::fresh().observe(WorkloadClass::PortBound, WorkloadClass::LatencyBound, 16);
assert_eq!(migrate, None);
assert_eq!(next.run, 1);
}
#[test]
fn a_window_reads_per_item_when_its_samples_carried_items() {
let leaves = 64u64;
let leaf_ns = 1_000_000u64;
let scaled = leaf_ns >> 8;
let dsum = leaf_ns * leaves;
let dsumsq = scaled * scaled * leaves;
assert_eq!(
window_reading(leaves, dsum, dsumsq, 0, 0),
(leaf_ns, Some(0))
);
let items = 16u64;
let per_item_sq = ((leaf_ns as u128 * leaf_ns as u128) / ((items as u128) << 16)) as u64;
let (mean, cv2) =
window_reading(leaves, dsum, dsumsq, leaves * items, per_item_sq * leaves);
assert_eq!(mean, leaf_ns / items);
let cv2 = cv2.expect("a 62,500 ns item is well above the unit");
assert!(
cv2 < 50,
"identical leaves read as a flat window; got {cv2}"
);
}
#[test]
fn a_window_under_the_unit_reads_its_mean_and_no_spread() {
let leaves = 64u64;
let leaf_ns = 200u64;
let dsum = leaf_ns * leaves;
assert_eq!(window_reading(leaves, dsum, 0, 0, 0), (leaf_ns, None));
assert_eq!(
window_reading(leaves, dsum, 0, leaves * 4, 0),
(leaf_ns / 4, None)
);
}
#[test]
fn a_window_decides_by_its_mean_where_the_mean_decides() {
let port_heavy = class_thresholds().port_heavy_ns.load(Ordering::Relaxed);
assert!(port_heavy > 1, "the heavy boundary is above one nanosecond");
let light = port_heavy - 1;
assert_eq!(
classify_window(light, None),
Some(classify_observed(light, 0))
);
assert_eq!(
classify_window(light, Some(900)),
Some(classify_observed(light, 900))
);
}
#[test]
fn a_heavy_window_with_no_readable_spread_decides_nothing() {
let heavy = class_thresholds().port_heavy_ns.load(Ordering::Relaxed) + 1_000;
assert_eq!(classify_window(heavy, None), None);
assert_eq!(
classify_window(heavy, Some(0)),
Some(WorkloadClass::Streaming)
);
assert_eq!(
classify_window(heavy, Some(10_000)),
Some(WorkloadClass::LatencyBound)
);
}
#[test]
fn a_class_survives_the_round_trip_a_migration_performs() {
for class in [
WorkloadClass::LatencyBound,
WorkloadClass::MemoryBound,
WorkloadClass::Streaming,
WorkloadClass::PortBound,
] {
let profile = class.to_dispatch_profile();
assert_eq!(
class_of_profile(profile_of_tag(tag_of_profile(profile))),
class
);
}
assert_eq!(
class_of_profile(profile_of_tag(tag_of_profile(
WorkloadClass::FineGrain.to_dispatch_profile()
))),
WorkloadClass::PortBound,
"the profile side has no FineGrain, so it collapses to PortBound"
);
}
}