use std::sync::{Arc, OnceLock};
use std::sync::atomic::{AtomicU64, Ordering::Relaxed};
use crate::sched::arena_local::{WorkerCtx, current_worker_ctx};
use crate::sched::arena_numa::NumaArena;
use crate::sched::job::{JobRef, NUMA_HINT_ANY, StackJob};
use crate::sched::latch::{JoinLatch, LockLatch, SpinLatch};
use crate::sched::sleep::Parker;
use crate::sched::plan::{JobPlan, SchedTier, pick_tier};
use crate::numa_topology::numa_topology;
pub fn join<A, B, RA, RB>(plan: &JobPlan, a: A, b: B) -> (RA, RB)
where
A: FnOnce() -> RA + Send,
B: FnOnce() -> RB + Send,
RA: Send,
RB: Send,
{
join_context(plan, move |_| a(), move |_| b())
}
pub fn join_context<A, B, RA, RB>(plan: &JobPlan, a: A, b: B) -> (RA, RB)
where
A: FnOnce(bool) -> RA + Send,
B: FnOnce(bool) -> RB + Send,
RA: Send,
RB: Send,
{
if plan.capped_to_caller() {
return inline_join_context(a, b);
}
let tier = pick_tier(plan, numa_topology());
match tier {
SchedTier::Inline => inline_join_context(a, b),
SchedTier::Local => local_join_context(plan, a, b),
SchedTier::Hierarchical | SchedTier::Federated => local_join_context(plan, a, b),
}
}
pub fn global_local_arena() -> &'static Arc<NumaArena> {
GLOBAL_ARENA.get_or_init(|| {
let per_node_override = pick_worker_count_per_node();
NumaArena::new(per_node_override)
})
}
pub fn global_local_arena_started() -> bool {
GLOBAL_ARENA.get().is_some()
}
static GLOBAL_ARENA: OnceLock<Arc<NumaArena>> = OnceLock::new();
fn pick_worker_count_per_node() -> Option<usize> {
if let Ok(v) = std::env::var("FLYNNEL_SCHED_WORKERS")
&& let Ok(n) = v.parse::<usize>()
&& n >= 1
{
return Some(n);
}
let physical_only = std::env::var("FLYNNEL_SCHED_PHYSICAL_ONLY")
.map(|v| v.eq_ignore_ascii_case("on") || v == "1" || v.eq_ignore_ascii_case("true"))
.unwrap_or(false);
let smt_explicitly_off = std::env::var("FLYNNEL_SCHED_SMT")
.map(|v| {
let lv = v.to_ascii_lowercase();
lv == "off" || lv == "0" || lv == "false"
})
.unwrap_or(false);
if physical_only || smt_explicitly_off {
return None;
}
let topo = numa_topology();
let cpus_node0 = topo.cpus_in_node(0).len();
Some(cpus_node0.max(1))
}
fn local_join_context<A, B, RA, RB>(plan: &JobPlan, a: A, b: B) -> (RA, RB)
where
A: FnOnce(bool) -> RA + Send,
B: FnOnce(bool) -> RB + Send,
RA: Send,
RB: Send,
{
let ctx_ptr = current_worker_ctx();
if !ctx_ptr.is_null() {
let ctx = unsafe { &*ctx_ptr };
return join_in_worker(ctx, plan, a, b, false);
}
external_dispatch(plan, a, b)
}
#[inline]
fn push_with_tier_hint(ctx: &WorkerCtx, job: JobRef, plan: &JobPlan) -> Result<(), JobRef> {
match plan.deque_tier_hint {
Some(tier) => ctx.try_push_tier(job, tier)?,
None => ctx.try_push(job)?,
}
if ctx.is_external_slot {
ctx.sleep.new_internal_jobs(ctx.stealers.len() as u32, false);
}
Ok(())
}
static JOIN_CALL_COUNT: AtomicU64 = AtomicU64::new(0);
static JOIN_A_BODY_NS: AtomicU64 = AtomicU64::new(0);
static JOIN_WAIT_NS: AtomicU64 = AtomicU64::new(0);
static JOIN_WAIT_STEAL_NS: AtomicU64 = AtomicU64::new(0);
static JOIN_WAIT_IDLE_NS: AtomicU64 = AtomicU64::new(0);
static DISPATCH_TRACE_ENABLED: OnceLock<bool> = OnceLock::new();
#[inline(always)]
fn dispatch_trace_enabled() -> bool {
*DISPATCH_TRACE_ENABLED.get_or_init(|| {
std::env::var_os("FLYNNEL_TRACE_DISPATCH").is_some()
})
}
#[inline(always)]
fn dispatch_tsc() -> u64 {
#[cfg(target_arch = "x86_64")]
unsafe {
std::arch::x86_64::_rdtsc()
}
#[cfg(not(target_arch = "x86_64"))]
{
std::time::Instant::now().elapsed().as_nanos() as u64
}
}
pub fn dispatch_trace_snapshot() -> (u64, u64, u64) {
(
JOIN_CALL_COUNT.swap(0, Relaxed),
JOIN_A_BODY_NS.swap(0, Relaxed),
JOIN_WAIT_NS.swap(0, Relaxed),
)
}
pub fn dispatch_trace_wait_snapshot() -> (u64, u64) {
(
JOIN_WAIT_STEAL_NS.swap(0, Relaxed),
JOIN_WAIT_IDLE_NS.swap(0, Relaxed),
)
}
static JOIN_PARKS: AtomicU64 = AtomicU64::new(0);
pub fn total_join_parks() -> u64 {
JOIN_PARKS.load(Relaxed)
}
fn park_for_half(latch: &JoinLatch) {
if !latch.get_sleepy() {
return;
}
if !latch.fall_asleep() {
return;
}
JOIN_PARKS.fetch_add(1, Relaxed);
let _unparked = latch.parker().park_until(|| latch.is_set());
latch.wake_up();
}
#[cold]
#[inline(never)]
fn wait_after_spin_budget(ctx: &WorkerCtx, latch: &JoinLatch) -> Option<u32> {
let reading = crate::sched::levers::join_park_oversubscribed();
if !ctx.is_external_slot
&& (crate::sched::levers::join_park()
|| (reading && crate::sched::oversubscription::recently()))
{
park_for_half(latch);
None
} else if reading {
let took = crate::sched::oversubscription::timed_yield();
crate::sched::trace::is_enabled().then(|| took.as_micros().min(u128::from(u32::MAX)) as u32)
} else {
let yield_started = crate::sched::trace::is_enabled().then(std::time::Instant::now);
std::thread::yield_now();
yield_started.map(|started| started.elapsed().as_micros().min(u128::from(u32::MAX)) as u32)
}
}
#[inline]
fn join_in_worker<A, B, RA, RB>(
ctx: &WorkerCtx,
plan: &JobPlan,
a: A,
b: B,
a_injected: bool,
) -> (RA, RB)
where
A: FnOnce(bool) -> RA + Send,
B: FnOnce(bool) -> RB + Send,
RA: Send,
RB: Send,
{
let job_b = StackJob::new(b, JoinLatch::new(&ctx.join_parker));
let job_b_ref = unsafe {
job_b.as_job_ref(
plan.k_outer,
plan.numa_hint.unwrap_or(NUMA_HINT_ANY as u32) as u8,
plan.variant,
)
};
let job_b_id = job_b_ref.id();
let sibling = ctx.index ^ 1;
let try_mailbox = plan.use_mailbox_routing
&& sibling < ctx.peer_mailboxes.len()
&& sibling < ctx.stealers.len()
&& ctx.peer_mailboxes[sibling].is_empty()
&& ctx.stealers[sibling][crate::sched::deque_tier::DequeTier::SmtLocal.idx()].is_empty();
let refused = if try_mailbox {
match ctx.push_to_mailbox(sibling, job_b_ref) {
Ok(()) => None,
Err(returned_job) => push_with_tier_hint(ctx, returned_job, plan).err(),
}
} else {
push_with_tier_hint(ctx, job_b_ref, plan).err()
};
if refused.is_some() {
let a_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| a(a_injected)));
let rb = unsafe { job_b.run_inline(false) };
return match a_result {
Ok(ra) => (ra, rb),
Err(payload) => {
drop(rb);
std::panic::resume_unwind(payload);
}
};
}
crate::sched::trace::emit(crate::sched::trace::TraceEvent::JoinPush, 0);
let traced = dispatch_trace_enabled();
let t_a_start = if traced { dispatch_tsc() } else { 0 };
let a_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| a(a_injected)));
let t_a_end = if traced { dispatch_tsc() } else { 0 };
crate::sched::trace::emit(crate::sched::trace::TraceEvent::JoinWaitBegin, 0);
let mut idle_since: Option<std::time::Instant> = None;
let mut last_yield_us: Option<u32> = None;
let spin_budget_ns = plan.effective_spin_before_yield_ns();
loop {
if job_b.latch.is_set() {
if let Some(us) = last_yield_us {
crate::sched::trace::emit(crate::sched::trace::TraceEvent::JoinLastYield, us);
}
crate::sched::trace::emit(crate::sched::trace::TraceEvent::JoinWaitEnd, 1);
if traced {
let t_wait_end = dispatch_tsc();
JOIN_CALL_COUNT.fetch_add(1, Relaxed);
JOIN_A_BODY_NS.fetch_add(t_a_end.wrapping_sub(t_a_start), Relaxed);
JOIN_WAIT_NS.fetch_add(t_wait_end.wrapping_sub(t_a_end), Relaxed);
}
let rb = unsafe { job_b.into_result() };
return match a_result {
Ok(ra) => (ra, rb),
Err(payload) => {
drop(rb);
std::panic::resume_unwind(payload);
}
};
}
if let Some(job) = ctx.find_work() {
if job.id() == job_b_id {
if let Some(us) = last_yield_us {
crate::sched::trace::emit(crate::sched::trace::TraceEvent::JoinLastYield, us);
}
crate::sched::trace::emit(crate::sched::trace::TraceEvent::JoinWaitEnd, 0);
if traced {
let t_wait_end = dispatch_tsc();
JOIN_CALL_COUNT.fetch_add(1, Relaxed);
JOIN_A_BODY_NS.fetch_add(t_a_end.wrapping_sub(t_a_start), Relaxed);
JOIN_WAIT_NS.fetch_add(t_wait_end.wrapping_sub(t_a_end), Relaxed);
}
let rb = unsafe { job_b.run_inline(false) };
return match a_result {
Ok(ra) => (ra, rb),
Err(payload) => {
drop(rb);
std::panic::resume_unwind(payload);
}
};
}
let t_steal_start = if traced { dispatch_tsc() } else { 0 };
unsafe { job.execute() };
if traced {
JOIN_WAIT_STEAL_NS.fetch_add(
dispatch_tsc().wrapping_sub(t_steal_start),
Relaxed,
);
}
} else {
let t_idle_start = if traced { dispatch_tsc() } else { 0 };
let idle_since = idle_since.get_or_insert_with(std::time::Instant::now);
if idle_since.elapsed().as_nanos() >= u128::from(spin_budget_ns) {
if let Some(us) = wait_after_spin_budget(ctx, &job_b.latch) {
last_yield_us = Some(us);
}
} else {
std::hint::spin_loop();
}
if traced {
JOIN_WAIT_IDLE_NS.fetch_add(
dispatch_tsc().wrapping_sub(t_idle_start),
Relaxed,
);
}
}
}
}
fn external_dispatch<A, B, RA, RB>(plan: &JobPlan, a: A, b: B) -> (RA, RB)
where
A: FnOnce(bool) -> RA + Send,
B: FnOnce(bool) -> RB + Send,
RA: Send,
RB: Send,
{
let arena = global_local_arena();
let _smt_guard_inline: Option<crate::sched::arena_local::SmtGuard>;
let _smt_guards_multi: Vec<crate::sched::arena_local::SmtGuard>;
if plan.effective_use_smt() {
if arena.is_single_numa() {
_smt_guard_inline = arena.try_acquire_smt_single();
_smt_guards_multi = Vec::new();
} else {
_smt_guard_inline = None;
_smt_guards_multi = arena.acquire_smt();
}
} else {
_smt_guard_inline = None;
_smt_guards_multi = Vec::new();
}
let arena_for_slot: Arc<crate::sched::arena_local::LocalArena> = arena
.single_node_arc()
.unwrap_or_else(|| arena.node_arc(plan.numa_hint));
if let Some(_guard) = arena_for_slot.try_claim_external_slot() {
let slot_ctx = _guard.ctx_ref();
let parker = Arc::new(Parker::new(if crate::sched::levers::slot_park_now() {
0
} else {
crate::sched::arena_local::LOCAL_SPIN_ROUNDS
}));
let job = StackJob::new(
move |_stolen: bool| -> (RA, RB) {
let primary_ctx_ptr = current_worker_ctx();
debug_assert!(
!primary_ctx_ptr.is_null(),
"wrap-and-park body must run on a worker thread"
);
let primary_ctx = unsafe { &*primary_ctx_ptr };
crate::sched::trace::emit(crate::sched::trace::TraceEvent::SlotJobStart, 0);
let out = join_in_worker(primary_ctx, plan, a, b, true);
crate::sched::trace::emit(crate::sched::trace::TraceEvent::SlotJobEnd, 0);
out
},
SpinLatch::new(parker.clone()),
);
unsafe {
let r = job.as_job_ref(
plan.k_outer,
plan.numa_hint.unwrap_or(NUMA_HINT_ANY as u32) as u8,
plan.variant,
);
slot_ctx.workers
[crate::sched::deque_tier::DequeTier::Public.idx()]
.push(r);
}
slot_ctx.sleep.new_internal_jobs(
slot_ctx.stealers.len() as u32,
false,
);
const SLOT_WAIT_SPIN: usize = 256;
crate::sched::trace::emit(crate::sched::trace::TraceEvent::SlotPush, 0);
let spin_budget_ns = plan.effective_spin_before_yield_ns();
let monitor_cycles = if crate::sched::levers::latch_monitor()
&& crate::sched::sleep::monitor_wait_available()
{
crate::sched::par_iter::installed_dispatch_cost_ns()
.saturating_mul(crate::sched::par_iter::tsc_per_ns_16())
/ 16
} else {
0
};
let wait_started = std::time::Instant::now();
let mut spun = 0usize;
let mut parked = false;
loop {
if job.latch.is_set() {
break;
}
let keep_spinning = spun < SLOT_WAIT_SPIN
|| wait_started.elapsed().as_nanos() < u128::from(spin_budget_ns);
if keep_spinning {
if monitor_cycles > 0 && spun >= SLOT_WAIT_SPIN {
if spun == SLOT_WAIT_SPIN {
crate::sched::sleep::note_latch_monitor_wait(true);
}
crate::sched::sleep::note_latch_monitor_arm();
unsafe {
crate::sched::sleep::monitor_wait_once(
job.latch.core.line(),
monitor_cycles,
|| job.latch.is_set(),
);
}
} else {
std::hint::spin_loop();
}
spun += 1;
continue;
}
if !job.latch.get_sleepy() {
spun = 0;
continue;
}
if job.latch.is_set() {
job.latch.wake_up();
break;
}
if !job.latch.fall_asleep() {
spun = 0;
continue;
}
let _unparked = parker.park_until(|| job.latch.is_set());
parked = true;
job.latch.wake_up();
spun = 0;
}
crate::sched::trace::emit(crate::sched::trace::TraceEvent::SlotWaitEnd, u32::from(!parked));
let result = unsafe { job.into_result() };
drop(_guard);
return result;
}
core::hint::cold_path();
let wrapper = StackJob::new(
move |_stolen: bool| -> (RA, RB) {
let ctx_ptr = current_worker_ctx();
debug_assert!(
!ctx_ptr.is_null(),
"external_dispatch wrapper must run on a worker thread"
);
let ctx = unsafe { &*ctx_ptr };
join_in_worker(ctx, plan, a, b, true)
},
LockLatch::new(),
);
unsafe {
let r = wrapper.as_job_ref(
plan.k_outer,
plan.numa_hint.unwrap_or(NUMA_HINT_ANY as u32) as u8,
plan.variant,
);
arena.submit(r, plan.numa_hint);
}
let predicted_ns = plan.estimated_per_item_ns
.map(|n| (n as u64).saturating_mul(plan.batch_size.max(1) as u64));
let spin_cycles = match predicted_ns {
Some(ns) => (ns / 3).clamp(1_000, 500_000) as usize,
None => 200_000,
};
wrapper.latch.wait_spin_then_monitor(spin_cycles);
unsafe { wrapper.into_result() }
}
#[inline]
fn inline_join_context<A, B, RA, RB>(a: A, b: B) -> (RA, RB)
where
A: FnOnce(bool) -> RA,
B: FnOnce(bool) -> RB,
{
let ra = a(false);
let rb = b(false);
(ra, rb)
}
pub fn join_default<A, B, RA, RB>(
k_outer: u8,
batch_size: u32,
a: A,
b: B,
) -> (RA, RB)
where
A: FnOnce() -> RA + Send,
B: FnOnce() -> RB + Send,
RA: Send,
RB: Send,
{
let plan = JobPlan::new(k_outer, batch_size);
join(&plan, a, b)
}
#[allow(dead_code)]
const _NUMA_HINT_ANY_REEXPORT: u8 = NUMA_HINT_ANY;
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, Ordering};
use crate::foundation::Variant;
use crate::sched::plan::HwClass;
#[test]
fn inline_join_returns_both_results() {
let plan = JobPlan::new(2, 1);
let (ra, rb) = join(&plan, || 10u32, || 20u32);
assert_eq!(ra, 10);
assert_eq!(rb, 20);
}
#[test]
fn inline_join_runs_a_before_b_when_serial() {
let order = Arc::new(AtomicU32::new(0));
let order_a = Arc::clone(&order);
let order_b = Arc::clone(&order);
let plan = JobPlan::new(2, 1).with_estimated_per_item_ns(10);
assert_eq!(pick_tier(&plan, numa_topology()), SchedTier::Inline);
let (ra, rb) = join(
&plan,
move || order_a.fetch_add(10, Ordering::SeqCst),
move || order_b.fetch_add(20, Ordering::SeqCst),
);
assert_eq!(ra, 0);
assert_eq!(rb, 10);
assert_eq!(order.load(Ordering::SeqCst), 30);
}
#[test]
fn inline_join_propagates_panic_from_a() {
let plan = JobPlan::new(2, 1);
let r = std::panic::catch_unwind(|| {
join::<_, _, u32, u32>(&plan, || panic!("a-side panic"), || 7u32)
});
assert!(r.is_err(), "a-side panic must propagate to caller");
}
#[test]
fn join_default_builds_plan_and_runs() {
let (ra, rb) = join_default(8, 1024, || 1u32, || 2u32);
assert_eq!((ra, rb), (1, 2));
}
#[test]
fn join_with_each_tier_band_dispatches_correctly() {
for k in [2u8, 6, 9, 13] {
let plan = JobPlan::new(k, 1024)
.with_hw_class(HwClass::Scalar)
.with_variant(Variant::Faithful);
let (ra, rb) = join(&plan, || k as u32, || (k as u32) << 1);
assert_eq!(ra, k as u32);
assert_eq!(rb, (k as u32) << 1);
}
}
#[test]
fn join_with_numa_hint_does_not_crash_single_node_host() {
let plan = JobPlan::new(6, 1024).with_numa_hint(0);
let (ra, rb) = join(&plan, || 1u32, || 2u32);
assert_eq!((ra, rb), (1, 2));
}
}