#![allow(clippy::missing_errors_doc)]
use core::sync::atomic::{AtomicU8, Ordering};
use crate::cpu_info::{Vendor, cpu_info};
use crate::dispatch_profile::DispatchProfile;
use crate::sched::plan::BisectVariant;
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Default)]
pub enum VariantRouting {
#[default]
Auto,
Default,
ComputeBatchAdaptive,
}
const TAG_AUTO: u8 = 0;
const TAG_DEFAULT: u8 = 1;
const TAG_COMPUTE_BATCH_ADAPTIVE: u8 = 2;
pub const COMPUTE_BATCH_LARGE_N: u32 = 50_000;
static ACTIVE_VARIANT_TAG: AtomicU8 = AtomicU8::new(TAG_AUTO);
#[unsafe(no_mangle)]
pub static __flynnel_marker_adaptive_variant_routing: u8 = 0;
#[inline]
pub fn cpuid_default_routing() -> VariantRouting {
match cpu_info().vendor {
Vendor::Amd => VariantRouting::ComputeBatchAdaptive,
Vendor::Intel | Vendor::Other => VariantRouting::Default,
}
}
#[inline]
pub fn active_variant_routing() -> VariantRouting {
match ACTIVE_VARIANT_TAG.load(Ordering::Acquire) {
TAG_DEFAULT => VariantRouting::Default,
TAG_COMPUTE_BATCH_ADAPTIVE => VariantRouting::ComputeBatchAdaptive,
_ => cpuid_default_routing(),
}
}
#[inline]
pub fn migrate_variant_routing(routing: VariantRouting) {
let tag = match routing {
VariantRouting::Auto => TAG_AUTO,
VariantRouting::Default => TAG_DEFAULT,
VariantRouting::ComputeBatchAdaptive => TAG_COMPUTE_BATCH_ADAPTIVE,
};
ACTIVE_VARIANT_TAG.store(tag, Ordering::Release);
}
#[inline]
pub fn pick_variant_for_profile(
profile: DispatchProfile,
batch_size: u32,
) -> Option<BisectVariant> {
match active_variant_routing() {
VariantRouting::Default => None,
VariantRouting::Auto => unreachable!("Auto resolves via active_variant_routing"),
VariantRouting::ComputeBatchAdaptive => {
if profile != DispatchProfile::PortBound {
return None;
}
if batch_size >= COMPUTE_BATCH_LARGE_N {
Some(BisectVariant::ProducerMaxLenWorkers)
} else {
Some(BisectVariant::RayonStyleReplenish)
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::{Mutex, MutexGuard, OnceLock};
fn global_test_lock() -> MutexGuard<'static, ()> {
static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
LOCK.get_or_init(|| Mutex::new(()))
.lock()
.expect("global_test_lock poisoned by prior test panic")
}
struct TestGuard {
_lock: MutexGuard<'static, ()>,
}
impl TestGuard {
fn new() -> Self {
let lock = global_test_lock();
migrate_variant_routing(VariantRouting::Auto);
Self { _lock: lock }
}
}
impl Drop for TestGuard {
fn drop(&mut self) {
migrate_variant_routing(VariantRouting::Auto);
}
}
#[test]
fn default_active_routing_resolves_from_cpuid() {
let _guard = TestGuard::new();
let active = active_variant_routing();
assert_eq!(active, cpuid_default_routing());
}
#[test]
fn migration_changes_active_routing() {
let _guard = TestGuard::new();
migrate_variant_routing(VariantRouting::Default);
assert_eq!(active_variant_routing(), VariantRouting::Default);
migrate_variant_routing(VariantRouting::ComputeBatchAdaptive);
assert_eq!(active_variant_routing(), VariantRouting::ComputeBatchAdaptive);
migrate_variant_routing(VariantRouting::Auto);
assert_eq!(active_variant_routing(), cpuid_default_routing());
}
#[test]
fn pick_variant_force_default_returns_none() {
let _guard = TestGuard::new();
migrate_variant_routing(VariantRouting::Default);
assert_eq!(pick_variant_for_profile(DispatchProfile::PortBound, 10_000), None);
assert_eq!(pick_variant_for_profile(DispatchProfile::PortBound, 100_000), None);
assert_eq!(pick_variant_for_profile(DispatchProfile::LatencyBound, 100_000), None);
}
#[test]
fn pick_variant_amd_compute_picks_by_batch_size() {
let _guard = TestGuard::new();
migrate_variant_routing(VariantRouting::ComputeBatchAdaptive);
assert_eq!(
pick_variant_for_profile(DispatchProfile::PortBound, 10_000),
Some(BisectVariant::RayonStyleReplenish)
);
assert_eq!(
pick_variant_for_profile(DispatchProfile::PortBound, COMPUTE_BATCH_LARGE_N),
Some(BisectVariant::ProducerMaxLenWorkers)
);
assert_eq!(
pick_variant_for_profile(DispatchProfile::PortBound, 100_000),
Some(BisectVariant::ProducerMaxLenWorkers)
);
assert_eq!(
pick_variant_for_profile(DispatchProfile::LatencyBound, 100_000),
None
);
assert_eq!(
pick_variant_for_profile(DispatchProfile::MemoryBound, 100_000),
None
);
}
#[test]
fn cpuid_default_matches_host_vendor() {
let info = cpu_info();
let expected = match info.vendor {
Vendor::Amd => VariantRouting::ComputeBatchAdaptive,
_ => VariantRouting::Default,
};
assert_eq!(cpuid_default_routing(), expected);
}
}