#![allow(clippy::missing_errors_doc)]
use core::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use std::sync::Arc;
use crate::sched::fcl_worker::{FclStash, FclSteal2, FclWorker, steal_via_stash as fcl_steal_via_stash};
use crate::sched::job::JobRef;
use crate::sched::k_gating::KGating;
use crate::sched::khl_local::KhlStealer;
use crate::sched::khl_worker::{KhlStash, KhlSteal2, KhlWorker, steal_via_stash as khl_steal_via_stash};
use crate::sched::fcl_local::FclStealer;
const ACTIVE_KHL: u32 = 0;
const ACTIVE_FCL: u32 = 1;
#[unsafe(no_mangle)]
pub static __flynnel_marker_adaptive_worker_push: u8 = 0;
#[unsafe(no_mangle)]
pub static __flynnel_marker_adaptive_worker_pop: u8 = 0;
pub struct AdaptiveWorker {
active: Arc<AtomicU32>,
khl: KhlWorker,
fcl: FclWorker,
inactive_dirty: Arc<AtomicBool>,
}
unsafe impl Send for AdaptiveWorker {}
pub struct AdaptiveStealer {
active: Arc<AtomicU32>,
khl: KhlStealer,
fcl: FclStealer,
inactive_dirty: Arc<AtomicBool>,
}
impl Clone for AdaptiveStealer {
fn clone(&self) -> Self {
Self {
active: Arc::clone(&self.active),
khl: self.khl.clone(),
fcl: self.fcl.clone(),
inactive_dirty: Arc::clone(&self.inactive_dirty),
}
}
}
impl AdaptiveStealer {
#[inline]
pub fn is_empty(&self) -> bool {
self.khl.is_empty() && self.fcl.is_empty()
}
#[inline]
pub fn khl_len(&self) -> usize {
self.khl.len()
}
}
pub struct AdaptiveStash {
khl: KhlStash,
fcl: FclStash,
}
impl AdaptiveStash {
#[inline]
pub fn empty() -> Self {
Self {
khl: KhlStash::empty(),
fcl: FclStash::empty(),
}
}
#[inline]
pub fn drain_one(&mut self) -> Option<JobRef> {
if let Some(j) = self.khl.drain_one() {
return Some(j);
}
self.fcl.drain_one()
}
#[inline]
pub fn is_empty(&self) -> bool {
self.khl.is_empty() && self.fcl.is_empty()
}
}
pub fn new_adaptive(
slot_capacity: usize,
initial_gating: KGating,
) -> (AdaptiveWorker, AdaptiveStealer) {
let (khl_w, khl_s) = KhlWorker::new(slot_capacity);
let (fcl_w, fcl_s) = FclWorker::new(slot_capacity);
let active = match initial_gating.resolved() {
KGating::CounterOnly => ACTIVE_FCL,
_ => ACTIVE_KHL,
};
let active_arc = Arc::new(AtomicU32::new(active));
let dirty_arc = Arc::new(AtomicBool::new(false));
(
AdaptiveWorker {
active: Arc::clone(&active_arc),
khl: khl_w,
fcl: fcl_w,
inactive_dirty: Arc::clone(&dirty_arc),
},
AdaptiveStealer {
active: active_arc,
khl: khl_s,
fcl: fcl_s,
inactive_dirty: dirty_arc,
},
)
}
impl AdaptiveWorker {
#[inline(always)]
pub fn push(&self, job: JobRef) {
match self.active.load(Ordering::Acquire) {
ACTIVE_KHL => self.khl.push(job),
_ => {
core::hint::cold_path();
self.fcl.push(job);
}
}
}
#[inline]
pub fn try_push(&self, job: JobRef) -> Result<(), JobRef> {
match self.active.load(Ordering::Acquire) {
ACTIVE_KHL => self.khl.try_push(job),
_ => self.fcl.try_push(job),
}
}
#[inline(always)]
pub fn push_burst(&self, job: JobRef) {
match self.active.load(Ordering::Acquire) {
ACTIVE_KHL => self.khl.push_burst(job),
_ => {
core::hint::cold_path();
self.fcl.push_burst(job);
}
}
}
#[inline]
pub fn flush(&self) {
match self.active.load(Ordering::Acquire) {
ACTIVE_KHL => self.khl.flush(),
_ => {
core::hint::cold_path();
self.fcl.flush();
}
}
}
#[inline]
pub fn pop(&self) -> Option<JobRef> {
let active = self.active.load(Ordering::Acquire);
let primary = if active == ACTIVE_KHL {
self.khl.pop()
} else {
core::hint::cold_path();
self.fcl.pop()
};
if primary.is_some() {
return primary;
}
if !self.inactive_dirty.load(Ordering::Acquire) {
return None;
}
core::hint::cold_path();
let fallback = if active == ACTIVE_KHL {
self.fcl.pop()
} else {
self.khl.pop()
};
if fallback.is_none() {
self.inactive_dirty.store(false, Ordering::Release);
}
fallback
}
#[inline]
pub fn is_empty(&self) -> bool {
self.khl.is_empty() && self.fcl.is_empty()
}
#[inline]
pub fn pending_items(&self) -> u8 {
match self.active.load(Ordering::Acquire) {
ACTIVE_KHL => self.khl.pending_items(),
_ => self.fcl.pending_items(),
}
}
#[inline]
pub fn migrate_to(&self, gating: KGating) {
let resolved = gating.resolved();
let target = match resolved {
KGating::CounterOnly => ACTIVE_FCL,
_ => ACTIVE_KHL,
};
let current = self.active.load(Ordering::Acquire);
if current == ACTIVE_KHL {
self.khl.flush();
} else {
self.fcl.flush();
}
if current != target {
self.inactive_dirty.store(true, Ordering::Release);
}
self.active.store(target, Ordering::Release);
}
#[inline]
pub fn active_gating(&self) -> KGating {
match self.active.load(Ordering::Acquire) {
ACTIVE_KHL => KGating::PerSlot,
_ => KGating::CounterOnly,
}
}
#[inline]
pub fn active_tag(&self) -> Arc<AtomicU32> {
Arc::clone(&self.active)
}
#[inline]
pub fn stealer(&self) -> AdaptiveStealer {
AdaptiveStealer {
active: Arc::clone(&self.active),
khl: self.khl.stealer(),
fcl: self.fcl.stealer(),
inactive_dirty: Arc::clone(&self.inactive_dirty),
}
}
}
pub enum AdaptiveSteal2<T> {
Success(T),
Empty,
Retry,
}
#[inline]
pub fn steal_via_stash(
stealer: &AdaptiveStealer,
stash: &mut AdaptiveStash,
) -> AdaptiveSteal2<JobRef> {
let active = stealer.active.load(Ordering::Acquire);
if active == ACTIVE_KHL {
match khl_steal_via_stash(&stealer.khl, &mut stash.khl) {
KhlSteal2::Success(j) => AdaptiveSteal2::Success(j),
KhlSteal2::Retry => AdaptiveSteal2::Retry,
KhlSteal2::Empty => {
if !stealer.inactive_dirty.load(Ordering::Acquire) {
return AdaptiveSteal2::Empty;
}
match fcl_steal_via_stash(&stealer.fcl, &mut stash.fcl) {
FclSteal2::Success(j) => AdaptiveSteal2::Success(j),
FclSteal2::Retry => AdaptiveSteal2::Retry,
FclSteal2::Empty => {
stealer.inactive_dirty.store(false, Ordering::Release);
AdaptiveSteal2::Empty
}
}
}
}
} else {
match fcl_steal_via_stash(&stealer.fcl, &mut stash.fcl) {
FclSteal2::Success(j) => AdaptiveSteal2::Success(j),
FclSteal2::Retry => AdaptiveSteal2::Retry,
FclSteal2::Empty => {
if !stealer.inactive_dirty.load(Ordering::Acquire) {
return AdaptiveSteal2::Empty;
}
match khl_steal_via_stash(&stealer.khl, &mut stash.khl) {
KhlSteal2::Success(j) => AdaptiveSteal2::Success(j),
KhlSteal2::Retry => AdaptiveSteal2::Retry,
KhlSteal2::Empty => {
stealer.inactive_dirty.store(false, Ordering::Release);
AdaptiveSteal2::Empty
}
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::foundation::Variant;
use crate::sched::job::StackJob;
use crate::sched::latch::CoreLatch;
#[test]
fn default_starts_on_khl() {
let (w, _s) = new_adaptive(4, KGating::PerSlot);
assert_eq!(w.active_gating(), KGating::PerSlot);
}
#[test]
fn explicit_counter_only_starts_on_fcl() {
let (w, _s) = new_adaptive(4, KGating::CounterOnly);
assert_eq!(w.active_gating(), KGating::CounterOnly);
}
#[test]
fn migrate_flips_active_tag() {
let (w, _s) = new_adaptive(4, KGating::PerSlot);
assert_eq!(w.active_gating(), KGating::PerSlot);
w.migrate_to(KGating::CounterOnly);
assert_eq!(w.active_gating(), KGating::CounterOnly);
w.migrate_to(KGating::PerSlot);
assert_eq!(w.active_gating(), KGating::PerSlot);
}
#[test]
fn push_pop_round_trip_on_khl() {
let (w, _s) = new_adaptive(4, KGating::PerSlot);
let j = StackJob::new(|_| 42u32, CoreLatch::new());
let r = unsafe { j.as_job_ref(4, 0, Variant::Faithful) };
w.push(r);
let popped = w.pop().expect("pop returns the pushed item");
unsafe { popped.execute() };
assert!(j.latch.is_set());
assert_eq!(unsafe { j.into_result() }, 42);
}
#[test]
fn push_pop_round_trip_on_fcl() {
let (w, _s) = new_adaptive(4, KGating::CounterOnly);
let j = StackJob::new(|_| 99u32, CoreLatch::new());
let r = unsafe { j.as_job_ref(4, 0, Variant::Faithful) };
w.push(r);
let popped = w.pop().expect("pop returns the pushed item");
unsafe { popped.execute() };
assert!(j.latch.is_set());
}
#[test]
fn migration_preserves_orphan_items() {
let (w, _s) = new_adaptive(4, KGating::PerSlot);
let j1 = StackJob::new(|_| 1u32, CoreLatch::new());
let r1 = unsafe { j1.as_job_ref(4, 0, Variant::Fast) };
w.push(r1);
w.migrate_to(KGating::CounterOnly);
let j2 = StackJob::new(|_| 2u32, CoreLatch::new());
let r2 = unsafe { j2.as_job_ref(4, 0, Variant::Fast) };
w.push(r2);
let p1 = w.pop().expect("first pop from active Fcl");
unsafe { p1.execute() };
let p2 = w.pop().expect("second pop from dormant KHL");
unsafe { p2.execute() };
assert!(j1.latch.is_set() && j2.latch.is_set());
assert!(w.pop().is_none());
}
#[test]
fn stealer_observes_migration() {
let (w, s) = new_adaptive(4, KGating::PerSlot);
let mut stash = AdaptiveStash::empty();
let j = StackJob::new(|_| 5u32, CoreLatch::new());
let r = unsafe { j.as_job_ref(4, 0, Variant::Fast) };
w.push(r);
match steal_via_stash(&s, &mut stash) {
AdaptiveSteal2::Success(jr) => {
unsafe { jr.execute() };
assert!(j.latch.is_set());
}
_ => panic!("expected Success"),
}
}
}