#![allow(clippy::missing_errors_doc)]
use core::cell::UnsafeCell;
use crate::sched::job::{CompactJobRef, JobRef};
use crate::sched::khl_local::{KHL_LINE_ITEMS, KhlBody, KhlSteal, KhlStealer, SchedKhlDeque, new_khl};
pub struct KhlStash {
n: u8,
k_outer: u8,
numa_hint: u8,
variant: u8,
items: [CompactJobRef; KHL_LINE_ITEMS - 1],
}
impl KhlStash {
#[inline]
pub fn empty() -> Self {
Self {
n: 0,
k_outer: 0,
numa_hint: 0,
variant: 0,
items: [CompactJobRef::null(), CompactJobRef::null()],
}
}
#[inline]
pub fn drain_one(&mut self) -> Option<JobRef> {
if self.n == 0 {
return None;
}
self.n -= 1;
let compact = self.items[self.n as usize];
Some(unsafe { compact.to_jobref(self.k_outer, self.numa_hint, self.variant) })
}
#[inline]
pub fn stash_extras(&mut self, body: KhlBody) -> Option<JobRef> {
let n = body.n_items as usize;
if n == 0 {
return None;
}
self.k_outer = body.k_outer;
self.numa_hint = body.numa_hint;
self.variant = body.variant;
let items = body.items_for_adapter();
self.items[..(n - 1)].copy_from_slice(&items[1..n]);
self.n = (n - 1) as u8;
let first = items[0];
Some(unsafe { first.to_jobref(body.k_outer, body.numa_hint, body.variant) })
}
#[inline]
pub fn is_empty(&self) -> bool {
self.n == 0
}
}
pub struct KhlWorker {
inner: SchedKhlDeque,
pop_stash: UnsafeCell<KhlStash>,
}
unsafe impl Send for KhlWorker {}
impl KhlWorker {
pub fn new(slot_capacity: usize) -> (Self, KhlStealer) {
let (inner, stealer) = new_khl(slot_capacity);
(
Self {
inner,
pop_stash: UnsafeCell::new(KhlStash::empty()),
},
stealer,
)
}
#[inline(always)]
pub fn push(&self, job: JobRef) {
self.inner.push_one(job);
}
#[inline]
pub fn try_push(&self, job: JobRef) -> Result<(), JobRef> {
self.inner.try_push_one(job)
}
#[inline(always)]
pub fn push_burst(&self, job: JobRef) {
self.inner.push(job);
}
#[inline]
pub fn pop(&self) -> Option<JobRef> {
let stash = unsafe { &mut *self.pop_stash.get() };
if let Some(j) = stash.drain_one() {
return Some(j);
}
match self.inner.pop() {
KhlSteal::Success(body) => stash.stash_extras(body),
KhlSteal::Empty | KhlSteal::Retry => None,
}
}
#[inline]
pub fn flush(&self) {
self.inner.flush();
}
#[inline]
pub fn pending_items(&self) -> u8 {
self.inner.accumulator_n_items()
}
#[inline]
pub fn is_empty(&self) -> bool {
let stash = unsafe { &*self.pop_stash.get() };
if !stash.is_empty() {
return false;
}
self.inner.accumulator_n_items() == 0 && self.inner.is_inner_empty()
}
#[inline]
pub fn stealer(&self) -> KhlStealer {
self.inner.stealer()
}
}
pub enum KhlSteal2<T> {
Success(T),
Empty,
Retry,
}
impl<T> KhlSteal2<T> {
#[inline]
pub fn ok(self) -> Option<T> {
match self {
KhlSteal2::Success(t) => Some(t),
_ => None,
}
}
}
#[inline]
pub fn steal_via_stash(
stealer: &KhlStealer,
stash: &mut KhlStash,
) -> KhlSteal2<JobRef> {
debug_assert!(stash.is_empty(), "stash must be drained before steal");
match stealer.steal() {
KhlSteal::Success(body) => match stash.stash_extras(body) {
Some(j) => KhlSteal2::Success(j),
None => KhlSteal2::Empty,
},
KhlSteal::Empty => KhlSteal2::Empty,
KhlSteal::Retry => KhlSteal2::Retry,
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::foundation::Variant;
use crate::sched::job::StackJob;
use crate::sched::latch::CoreLatch;
#[test]
fn pop_serves_three_from_one_inner_burst() {
let (w, _s) = KhlWorker::new(4);
let j1 = StackJob::new(|_| 1u32, CoreLatch::new());
let j2 = StackJob::new(|_| 2u32, CoreLatch::new());
let j3 = StackJob::new(|_| 3u32, CoreLatch::new());
let r1 = unsafe { j1.as_job_ref(4, 0, Variant::Fast) };
let r2 = unsafe { j2.as_job_ref(4, 0, Variant::Fast) };
let r3 = unsafe { j3.as_job_ref(4, 0, Variant::Fast) };
w.push_burst(r1);
w.push_burst(r2);
w.push_burst(r3);
let p1 = w.pop().expect("first pop");
unsafe { p1.execute() };
let p2 = w.pop().expect("second pop");
unsafe { p2.execute() };
let p3 = w.pop().expect("third pop");
unsafe { p3.execute() };
assert!(j1.latch.is_set() && j2.latch.is_set() && j3.latch.is_set());
assert!(w.pop().is_none());
}
#[test]
fn push_one_always_flushes_for_join_pattern() {
let (w, s) = KhlWorker::new(4);
let j = StackJob::new(|_| 7u32, CoreLatch::new());
let r = unsafe { j.as_job_ref(8, 0, Variant::Faithful) };
w.push(r);
match s.steal() {
KhlSteal::Success(body) => {
assert_eq!(body.n_items, 1);
unsafe { body.execute_all_lifo() };
assert!(j.latch.is_set());
}
other => panic!("expected Success, got {other:?}"),
}
}
#[test]
fn burst_then_flush_publishes_partial() {
let (w, s) = KhlWorker::new(4);
let j = StackJob::new(|_| 7u32, CoreLatch::new());
let r = unsafe { j.as_job_ref(8, 0, Variant::Faithful) };
w.push_burst(r);
assert!(matches!(s.steal(), KhlSteal::Empty));
w.flush();
match s.steal() {
KhlSteal::Success(body) => {
assert_eq!(body.n_items, 1);
unsafe { body.execute_all_lifo() };
assert!(j.latch.is_set());
}
other => panic!("expected Success, got {other:?}"),
}
}
#[test]
#[allow(clippy::type_complexity)]
fn steal_via_stash_full_round_trip() {
let (w, s) = KhlWorker::new(4);
let mut stash = KhlStash::empty();
let jobs: [StackJob<CoreLatch, fn(bool) -> u32, u32>; 6] = [
StackJob::new((|_| 1) as fn(bool) -> u32, CoreLatch::new()),
StackJob::new((|_| 2) as fn(bool) -> u32, CoreLatch::new()),
StackJob::new((|_| 3) as fn(bool) -> u32, CoreLatch::new()),
StackJob::new((|_| 4) as fn(bool) -> u32, CoreLatch::new()),
StackJob::new((|_| 5) as fn(bool) -> u32, CoreLatch::new()),
StackJob::new((|_| 6) as fn(bool) -> u32, CoreLatch::new()),
];
for j in &jobs {
let r = unsafe { j.as_job_ref(4, 0, Variant::Fast) };
w.push_burst(r);
}
let mut executed = 0usize;
loop {
if let Some(j) = stash.drain_one() {
unsafe { j.execute() };
executed += 1;
continue;
}
match steal_via_stash(&s, &mut stash) {
KhlSteal2::Success(j) => {
unsafe { j.execute() };
executed += 1;
}
KhlSteal2::Empty => break,
KhlSteal2::Retry => continue,
}
}
assert_eq!(executed, 6);
for j in &jobs {
assert!(j.latch.is_set());
}
}
}