use core::any::Any;
use core::cell::UnsafeCell;
use std::panic;
use crate::foundation::Variant;
use crate::sched::latch::Latch;
pub const NUMA_HINT_ANY: u8 = 0xFF;
pub(crate) trait Job {
unsafe fn execute(this: *const ());
}
#[allow(dead_code, reason = "`reserved` is read by the K_inner=3 slot path indirectly through job.compact(); the other tag bytes are consumed by SchedFclDeque/SchedKhlDeque push to populate the shared slot header. Marked allow rather than expect because the dead_code lint may fire under future refactors that remove specific tag consumers.")]
pub struct JobRef {
pointer: *const (),
execute_fn: unsafe fn(*const ()),
pub k_outer: u8,
pub numa_hint: u8,
pub variant: u8,
pub reserved: u8,
}
unsafe impl Send for JobRef {}
unsafe impl Sync for JobRef {}
impl JobRef {
pub(crate) unsafe fn new<T>(
data: *const T,
k_outer: u8,
numa_hint: u8,
variant: Variant,
) -> Self
where
T: Job,
{
Self {
pointer: data as *const (),
execute_fn: <T as Job>::execute,
k_outer,
numa_hint,
variant: variant_to_tag(variant),
reserved: 0,
}
}
#[inline]
pub unsafe fn execute(self) {
unsafe { (self.execute_fn)(self.pointer) }
}
#[inline]
pub(crate) fn id(&self) -> usize {
self.pointer as usize
}
#[inline]
pub(crate) fn data_ptr(&self) -> *const () {
self.pointer
}
#[cfg(test)]
#[inline]
pub fn variant_decoded(&self) -> Variant {
tag_to_variant(self.variant)
}
#[inline]
pub unsafe fn from_raw_parts(
pointer: *const (),
execute_fn: unsafe fn(*const ()),
k_outer: u8,
numa_hint: u8,
variant: u8,
) -> Self {
Self {
pointer,
execute_fn,
k_outer,
numa_hint,
variant,
reserved: 0,
}
}
#[inline]
pub(crate) fn compact(&self) -> CompactJobRef {
CompactJobRef {
pointer: self.pointer,
execute_fn: self.execute_fn,
}
}
}
#[derive(Copy, Clone)]
pub struct CompactJobRef {
pointer: *const (),
execute_fn: unsafe fn(*const ()),
}
unsafe impl Send for CompactJobRef {}
unsafe impl Sync for CompactJobRef {}
impl CompactJobRef {
pub const fn null() -> Self {
unsafe fn never_called(_: *const ()) {
panic!("CompactJobRef::null() must never be executed");
}
Self {
pointer: std::ptr::null(),
execute_fn: never_called,
}
}
#[inline]
pub unsafe fn to_jobref(self, k_outer: u8, numa_hint: u8, variant: u8) -> JobRef {
JobRef {
pointer: self.pointer,
execute_fn: self.execute_fn,
k_outer,
numa_hint,
variant,
reserved: 0,
}
}
#[inline]
pub unsafe fn execute(self) {
unsafe { (self.execute_fn)(self.pointer) }
}
}
#[inline]
fn variant_to_tag(v: Variant) -> u8 {
match v {
Variant::Correct => 0,
Variant::Faithful => 1,
Variant::Fast => 2,
}
}
#[cfg(test)]
#[inline]
fn tag_to_variant(tag: u8) -> Variant {
match tag {
0 => Variant::Correct,
2 => Variant::Fast,
_ => Variant::Faithful,
}
}
pub(crate) enum JobResult<T> {
None,
Ok(T),
Panic(Box<dyn Any + Send + 'static>),
}
impl<T> JobResult<T> {
fn capture<F: FnOnce(bool) -> T>(func: F, stolen: bool) -> Self {
match panic::catch_unwind(panic::AssertUnwindSafe(|| func(stolen))) {
Ok(value) => JobResult::Ok(value),
Err(payload) => JobResult::Panic(payload),
}
}
pub(crate) fn into_return_value(self) -> T {
match self {
JobResult::None => panic!("JobResult was never written"),
JobResult::Ok(value) => value,
JobResult::Panic(payload) => panic::resume_unwind(payload),
}
}
}
#[repr(C, align(64))]
pub(crate) struct StackJob<L, F, R>
where
L: Latch + Sync,
F: FnOnce(bool) -> R + Send,
R: Send,
{
func: UnsafeCell<Option<F>>,
result: UnsafeCell<JobResult<R>>,
pub(crate) latch: L,
}
impl<L, F, R> StackJob<L, F, R>
where
L: Latch + Sync,
F: FnOnce(bool) -> R + Send,
R: Send,
{
pub(crate) fn new(func: F, latch: L) -> Self {
Self {
latch,
func: UnsafeCell::new(Some(func)),
result: UnsafeCell::new(JobResult::None),
}
}
pub(crate) unsafe fn as_job_ref(
&self,
k_outer: u8,
numa_hint: u8,
variant: Variant,
) -> JobRef {
unsafe { JobRef::new(self, k_outer, numa_hint, variant) }
}
pub(crate) unsafe fn run_inline(self, stolen: bool) -> R {
let func = unsafe { (*self.func.get()).take() }.expect("closure already taken");
func(stolen)
}
pub(crate) unsafe fn into_result(self) -> R {
let result = unsafe { core::ptr::read(self.result.get()) };
unsafe { core::ptr::write(self.result.get(), JobResult::None) };
result.into_return_value()
}
}
impl<L, F, R> Job for StackJob<L, F, R>
where
L: Latch + Sync,
F: FnOnce(bool) -> R + Send,
R: Send,
{
unsafe fn execute(this: *const ()) {
let this = unsafe { &*(this as *const Self) };
let func = unsafe { (*this.func.get()).take() }.expect("closure already taken");
let result = JobResult::capture(func, true );
unsafe { core::ptr::write(this.result.get(), result) };
crate::sched::trace::emit(crate::sched::trace::TraceEvent::LatchSet, 0);
unsafe { Latch::set(&this.latch) };
}
}
pub(crate) struct StackJobShared<L, F, R>
where
L: Latch + Sync,
F: FnOnce(bool) -> R + Send,
R: Send,
{
pub(crate) latch: std::sync::Arc<L>,
func: UnsafeCell<Option<F>>,
result: UnsafeCell<JobResult<R>>,
}
impl<L, F, R> StackJobShared<L, F, R>
where
L: Latch + Sync,
F: FnOnce(bool) -> R + Send,
R: Send,
{
pub(crate) fn new(func: F, latch: std::sync::Arc<L>) -> Self {
Self {
latch,
func: UnsafeCell::new(Some(func)),
result: UnsafeCell::new(JobResult::None),
}
}
pub(crate) unsafe fn as_job_ref(
&self,
k_outer: u8,
numa_hint: u8,
variant: Variant,
) -> JobRef {
unsafe { JobRef::new(self, k_outer, numa_hint, variant) }
}
pub(crate) unsafe fn into_result(self) -> R {
let result = unsafe { core::ptr::read(self.result.get()) };
unsafe { core::ptr::write(self.result.get(), JobResult::None) };
result.into_return_value()
}
}
impl<L, F, R> Job for StackJobShared<L, F, R>
where
L: Latch + Sync,
F: FnOnce(bool) -> R + Send,
R: Send,
{
unsafe fn execute(this: *const ()) {
let this = unsafe { &*(this as *const Self) };
let func = unsafe { (*this.func.get()).take() }.expect("closure already taken");
let result = JobResult::capture(func, true);
unsafe { core::ptr::write(this.result.get(), result) };
let latch_ptr: *const L = std::sync::Arc::as_ptr(&this.latch);
unsafe { Latch::set(latch_ptr) };
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::foundation::Variant;
use crate::sched::latch::CoreLatch;
#[test]
fn jobref_tag_round_trip_variant() {
for v in [Variant::Correct, Variant::Faithful, Variant::Fast] {
let job = StackJob::new(|_stolen| 42u32, CoreLatch::new());
let r = unsafe { job.as_job_ref(8, 1, v) };
assert_eq!(r.variant_decoded(), v, "variant {v:?} should round-trip");
assert_eq!(r.k_outer, 8);
assert_eq!(r.numa_hint, 1);
unsafe { r.execute() };
assert!(job.latch.is_set());
}
}
#[test]
fn jobref_size_is_two_pointers_plus_tags() {
let sz = core::mem::size_of::<JobRef>();
let align = core::mem::align_of::<JobRef>();
assert!(sz <= 32, "JobRef should be small; got {sz} bytes");
assert!(align >= core::mem::align_of::<*const ()>());
}
#[test]
fn stack_job_executes_closure_and_sets_latch() {
let job = StackJob::new(|_stolen| 0x1234_5678u32, CoreLatch::new());
let r = unsafe { job.as_job_ref(8, NUMA_HINT_ANY, Variant::Faithful) };
assert!(!job.latch.is_set(), "latch starts un-set");
unsafe { r.execute() };
assert!(job.latch.is_set(), "execute must set the latch");
let value = unsafe { job.into_result() };
assert_eq!(value, 0x1234_5678);
}
#[test]
fn stack_job_executes_with_stolen_flag_true() {
let job = StackJob::new(|stolen| stolen, CoreLatch::new());
let r = unsafe { job.as_job_ref(2, 0, Variant::Fast) };
unsafe { r.execute() };
let value = unsafe { job.into_result() };
assert!(value, "JobRef::execute must pass stolen = true");
}
#[test]
fn stack_job_shared_executes_and_sets_shared_latch() {
use crate::sched::latch::CountLatch;
use crate::sched::sleep::Parker;
let parker = std::sync::Arc::new(Parker::new(0));
let latch = std::sync::Arc::new(CountLatch::new(3, parker));
let jobs: Vec<StackJobShared<CountLatch, _, u32>> = (0..3u32)
.map(|i| StackJobShared::new(move |_stolen| i * 10, latch.clone()))
.collect();
assert!(!latch.is_set());
assert_eq!(latch.outstanding(), 3);
let r0 = unsafe { jobs[0].as_job_ref(8, NUMA_HINT_ANY, Variant::Faithful) };
unsafe { r0.execute() };
assert!(!latch.is_set(), "latch must NOT set after 1/3 decrements");
assert_eq!(latch.outstanding(), 2);
let r1 = unsafe { jobs[1].as_job_ref(8, NUMA_HINT_ANY, Variant::Faithful) };
unsafe { r1.execute() };
assert!(!latch.is_set(), "latch must NOT set after 2/3 decrements");
assert_eq!(latch.outstanding(), 1);
let r2 = unsafe { jobs[2].as_job_ref(8, NUMA_HINT_ANY, Variant::Faithful) };
unsafe { r2.execute() };
assert!(latch.is_set(), "latch MUST set on final decrement");
assert_eq!(latch.outstanding(), 0);
let mut iter = jobs.into_iter();
let v0 = unsafe { iter.next().unwrap().into_result() };
let v1 = unsafe { iter.next().unwrap().into_result() };
let v2 = unsafe { iter.next().unwrap().into_result() };
assert_eq!((v0, v1, v2), (0, 10, 20));
}
#[test]
fn run_inline_passes_stolen_flag_false() {
let job = StackJob::new(|stolen| stolen, CoreLatch::new());
let value = unsafe { job.run_inline(false) };
assert!(!value, "run_inline must pass stolen = false");
}
#[test]
#[cfg(panic = "unwind")]
fn stack_job_captures_panic_for_parent_resume() {
let job = StackJob::new(|_stolen| -> u32 {
panic!("intentional test panic from worker closure");
}, CoreLatch::new());
let r = unsafe { job.as_job_ref(2, NUMA_HINT_ANY, Variant::Faithful) };
unsafe { r.execute() };
assert!(job.latch.is_set(), "latch must be set even on panic");
let payload = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| unsafe {
job.into_result()
}));
assert!(payload.is_err(), "into_result must resume the captured panic");
}
#[test]
fn variant_tag_decodes_unknown_to_faithful() {
assert_eq!(tag_to_variant(0), Variant::Correct);
assert_eq!(tag_to_variant(1), Variant::Faithful);
assert_eq!(tag_to_variant(2), Variant::Fast);
assert_eq!(tag_to_variant(3), Variant::Faithful);
assert_eq!(tag_to_variant(0xFF), Variant::Faithful);
}
}