#![allow(clippy::missing_errors_doc)]
use core::sync::atomic::{AtomicU8, Ordering};
use crate::cpu_info::{Vendor, cpu_info};
use crate::dispatch_profile::DispatchProfile;
use crate::sched::plan::BisectVariant;
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Default)]
pub enum VariantRouting {
#[default]
Auto,
Default,
ComputeBatchAdaptive,
}
const TAG_AUTO: u8 = 0;
const TAG_DEFAULT: u8 = 1;
const TAG_COMPUTE_BATCH_ADAPTIVE: u8 = 2;
pub const COMPUTE_BATCH_LARGE_N: u32 = 50_000;
static ACTIVE_VARIANT_TAG: AtomicU8 = AtomicU8::new(ACTIVE_VARIANT_TAG_DEFAULT);
const ACTIVE_VARIANT_TAG_DEFAULT: u8 = TAG_AUTO;
#[unsafe(no_mangle)]
pub static __flynnel_marker_adaptive_variant_routing: u8 = 0;
#[inline]
pub fn cpuid_default_routing() -> VariantRouting {
match cpu_info().vendor {
Vendor::Amd => VariantRouting::ComputeBatchAdaptive,
Vendor::Intel | Vendor::Other => VariantRouting::Default,
}
}
#[inline]
pub fn active_variant_routing() -> VariantRouting {
match routing_of_tag(ACTIVE_VARIANT_TAG.load(Ordering::Acquire)) {
VariantRouting::Auto => cpuid_default_routing(),
named => named,
}
}
#[inline]
const fn routing_of_tag(tag: u8) -> VariantRouting {
match tag {
TAG_DEFAULT => VariantRouting::Default,
TAG_COMPUTE_BATCH_ADAPTIVE => VariantRouting::ComputeBatchAdaptive,
_ => VariantRouting::Auto,
}
}
#[inline]
const fn tag_of_routing(routing: VariantRouting) -> u8 {
match routing {
VariantRouting::Auto => TAG_AUTO,
VariantRouting::Default => TAG_DEFAULT,
VariantRouting::ComputeBatchAdaptive => TAG_COMPUTE_BATCH_ADAPTIVE,
}
}
#[inline]
pub fn migrate_variant_routing(routing: VariantRouting) {
ACTIVE_VARIANT_TAG.store(tag_of_routing(routing), Ordering::Release);
}
#[inline]
pub fn pick_variant_for_profile(
profile: DispatchProfile,
batch_size: u32,
) -> Option<BisectVariant> {
pick_variant_under(active_variant_routing(), profile, batch_size)
}
#[inline]
fn pick_variant_under(
routing: VariantRouting,
profile: DispatchProfile,
batch_size: u32,
) -> Option<BisectVariant> {
let resolved = match routing {
VariantRouting::Auto => cpuid_default_routing(),
named => named,
};
match resolved {
VariantRouting::Default => None,
VariantRouting::Auto => unreachable!("Auto resolved above"),
VariantRouting::ComputeBatchAdaptive => {
if profile != DispatchProfile::PortBound {
return None;
}
if batch_size >= COMPUTE_BATCH_LARGE_N {
Some(BisectVariant::ProducerMaxLenWorkers)
} else {
Some(BisectVariant::RayonStyleReplenish)
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn every_routing_survives_its_tag() {
for routing in [
VariantRouting::Auto,
VariantRouting::Default,
VariantRouting::ComputeBatchAdaptive,
] {
assert_eq!(routing_of_tag(tag_of_routing(routing)), routing);
}
assert_eq!(routing_of_tag(TAG_AUTO), VariantRouting::Auto);
assert_eq!(routing_of_tag(200), VariantRouting::Auto);
}
#[test]
fn the_cell_starts_on_the_cpuid_default() {
assert_eq!(
routing_of_tag(ACTIVE_VARIANT_TAG_DEFAULT),
VariantRouting::Auto,
"the cell's initial tag defers to CPUID"
);
assert_eq!(
pick_variant_under(VariantRouting::Auto, DispatchProfile::PortBound, 10_000),
pick_variant_under(cpuid_default_routing(), DispatchProfile::PortBound, 10_000),
"Auto picks what this host's CPUID default picks"
);
}
#[test]
fn pick_variant_force_default_returns_none() {
for (profile, batch) in [
(DispatchProfile::PortBound, 10_000),
(DispatchProfile::PortBound, 100_000),
(DispatchProfile::LatencyBound, 100_000),
] {
assert_eq!(
pick_variant_under(VariantRouting::Default, profile, batch),
None
);
}
}
#[test]
fn pick_variant_amd_compute_picks_by_batch_size() {
let routing = VariantRouting::ComputeBatchAdaptive;
assert_eq!(
pick_variant_under(routing, DispatchProfile::PortBound, 10_000),
Some(BisectVariant::RayonStyleReplenish)
);
assert_eq!(
pick_variant_under(routing, DispatchProfile::PortBound, COMPUTE_BATCH_LARGE_N),
Some(BisectVariant::ProducerMaxLenWorkers)
);
assert_eq!(
pick_variant_under(routing, DispatchProfile::PortBound, 100_000),
Some(BisectVariant::ProducerMaxLenWorkers)
);
assert_eq!(
pick_variant_under(routing, DispatchProfile::LatencyBound, 100_000),
None
);
assert_eq!(
pick_variant_under(routing, DispatchProfile::MemoryBound, 100_000),
None
);
}
#[test]
fn cpuid_default_matches_host_vendor() {
let info = cpu_info();
let expected = match info.vendor {
Vendor::Amd => VariantRouting::ComputeBatchAdaptive,
_ => VariantRouting::Default,
};
assert_eq!(cpuid_default_routing(), expected);
}
}