1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
//! WorkerCtx-shaped adapter over [`crate::sched::fcl_local`].
//!
//! Mirrors [`crate::sched::khl_worker::KhlWorker`] but wraps
//! [`crate::sched::fcl_local::SchedFclDeque`] (counter-only
//! Chase-Lev family at K_inner=3). Used by the adaptive WorkerCtx
//! when the host's K_gating calibration prefers CounterOnly
//! (smaller-store-buffer cores).
//!
//! The K_inner=3 batching benefit is identical between Fcl and
//! KHL; the K_gating axis chooses how the publication signal is
//! transported (one shared bottom counter for Fcl, distributed
//! per-slot seq atomics for KHL).
#![allow(clippy::missing_errors_doc)]
use core::cell::UnsafeCell;
use crate::sched::chase_lev_local;
use crate::sched::fcl_local::{FclStealer, PoppedSlot, SchedFclDeque, new_fcl};
use crate::sched::job::{CompactJobRef, JobRef};
use crate::sched::khl_local::KHL_LINE_ITEMS;
/// Per-worker batch stash for the Fcl adapter (mirror of KhlStash
/// in shape; 0..=2 leftover items after a PoppedSlot drain).
pub struct FclStash {
n: u8,
k_outer: u8,
numa_hint: u8,
variant: u8,
items: [CompactJobRef; KHL_LINE_ITEMS - 1],
}
impl FclStash {
/// Fresh empty stash.
#[inline]
pub fn empty() -> Self {
Self {
n: 0,
k_outer: 0,
numa_hint: 0,
variant: 0,
items: [CompactJobRef::null(), CompactJobRef::null()],
}
}
/// Drain one item LIFO.
#[inline]
pub fn drain_one(&mut self) -> Option<JobRef> {
if self.n == 0 {
return None;
}
self.n -= 1;
let compact = self.items[self.n as usize];
// SAFETY: compact came from a fresh PoppedSlot; once-only
// preserved across drain calls.
Some(unsafe { compact.to_jobref(self.k_outer, self.numa_hint, self.variant) })
}
/// Absorb a popped/stolen PoppedSlot. Returns first item; rest
/// stash for subsequent drains.
#[inline]
pub fn stash_extras(&mut self, slot: PoppedSlot) -> Option<JobRef> {
let n = slot.n_items as usize;
if n == 0 {
return None;
}
self.k_outer = slot.k_outer;
self.numa_hint = slot.numa_hint;
self.variant = slot.variant;
// The items in PoppedSlot are pub(crate); use the slot's
// execute_all_lifo-compatible access path. Here we copy
// items[0..n] out via a small temp because the items field
// is private to fcl_local. The adapter is in flynnel so
// pub(crate) is visible.
let items_arr = slot.compact_items_for_adapter();
self.items[..(n - 1)].copy_from_slice(&items_arr[1..n]);
self.n = (n - 1) as u8;
let first = items_arr[0];
// SAFETY: items[0] from fresh slot; once-only preserved.
Some(unsafe { first.to_jobref(slot.k_outer, slot.numa_hint, slot.variant) })
}
/// True when stash holds no items.
#[inline]
pub fn is_empty(&self) -> bool {
self.n == 0
}
}
/// Owner-side Fcl adapter. Wraps a [`SchedFclDeque`] with an
/// internal pop-stash so callers see a JobRef-shaped API
/// (mirrors `KhlWorker` so the two are interchangeable at the
/// WorkerCtx tagged-enum dispatch site).
pub struct FclWorker {
inner: SchedFclDeque,
pop_stash: UnsafeCell<FclStash>,
}
// SAFETY: only the owner thread accesses pop_stash; inner handles
// its own Send/Sync via Chase-Lev.
unsafe impl Send for FclWorker {}
impl FclWorker {
/// Construct a new Fcl-backed owner adapter + paired stealer.
/// Capacity is in SLOTS; each slot carries up to 3 jobs.
pub fn new(slot_capacity: usize) -> (Self, FclStealer) {
let (inner, stealer) = new_fcl(slot_capacity);
(
Self {
inner,
pop_stash: UnsafeCell::new(FclStash::empty()),
},
stealer,
)
}
/// Push one job with auto-flush (single-push API for join sites).
#[inline(always)]
pub fn push(&self, job: JobRef) {
self.inner.push(job);
self.inner.flush();
}
/// Non-blocking single push; `Err(job)` when the inner deque is
/// full instead of spinning for a thief.
#[inline]
pub fn try_push(&self, job: JobRef) -> Result<(), JobRef> {
self.inner.try_push_one(job)
}
/// Push without auto-flush (burst API for cooperative fan-out).
#[inline(always)]
pub fn push_burst(&self, job: JobRef) {
self.inner.push(job);
}
/// Flush any partially-filled accumulator slot.
#[inline]
pub fn flush(&self) {
self.inner.flush();
}
/// Pop one job. Drains pop-stash first; if empty, fetches a
/// slot from the inner Fcl and stashes the extras.
#[inline]
pub fn pop(&self) -> Option<JobRef> {
// SAFETY: owner-private pop_stash.
let stash = unsafe { &mut *self.pop_stash.get() };
if let Some(j) = stash.drain_one() {
return Some(j);
}
match self.inner.pop() {
Some(slot) => stash.stash_extras(slot),
None => None,
}
}
/// Approximate is-empty hint.
#[inline]
pub fn is_empty(&self) -> bool {
// SAFETY: owner-private read.
let stash = unsafe { &*self.pop_stash.get() };
stash.is_empty()
// Inner emptiness is not exposed; conservative hint.
}
/// Number of items currently buffered in the accumulator
/// (0..=3). Used by WorkerCtx::flush_all for the JEC wake
/// broadcast count.
#[inline]
pub fn pending_items(&self) -> u8 {
// Accumulator size isn't exposed on SchedFclDeque; the
// adapter conservatively reports 0. The flush_all path
// uses the burst counter as the canonical pending-count
// for JEC wake purposes; this method exists for API
// parity with KhlWorker.
0
}
/// Clone a fresh thief handle.
#[inline]
pub fn stealer(&self) -> FclStealer {
self.inner.stealer()
}
}
/// Outcome of a thief-side Fcl steal at the WorkerCtx API level.
/// Mirrors KhlSteal2.
pub enum FclSteal2<T> {
/// Got a job; extras stashed.
Success(T),
/// Deque empty.
Empty,
/// CAS-loss; caller should retry.
Retry,
}
/// Thief-side helper: steal one slot via Fcl + absorb extras into
/// caller's stash. Mirrors `khl_worker::steal_via_stash`.
#[inline]
pub fn steal_via_stash(
stealer: &FclStealer,
stash: &mut FclStash,
) -> FclSteal2<JobRef> {
debug_assert!(stash.is_empty(), "stash must be drained before steal");
match stealer.steal() {
chase_lev_local::Steal::Success(slot) => match stash.stash_extras(slot) {
Some(j) => FclSteal2::Success(j),
None => FclSteal2::Empty,
},
chase_lev_local::Steal::Empty => FclSteal2::Empty,
chase_lev_local::Steal::Retry => FclSteal2::Retry,
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::foundation::Variant;
use crate::sched::job::StackJob;
use crate::sched::latch::CoreLatch;
#[test]
fn pop_serves_three_from_one_inner_burst() {
let (w, _s) = FclWorker::new(4);
let j1 = StackJob::new(|_| 1u32, CoreLatch::new());
let j2 = StackJob::new(|_| 2u32, CoreLatch::new());
let j3 = StackJob::new(|_| 3u32, CoreLatch::new());
let r1 = unsafe { j1.as_job_ref(4, 0, Variant::Fast) };
let r2 = unsafe { j2.as_job_ref(4, 0, Variant::Fast) };
let r3 = unsafe { j3.as_job_ref(4, 0, Variant::Fast) };
w.push_burst(r1);
w.push_burst(r2);
w.push_burst(r3);
// push_burst auto-flushes at 3 via SchedFclDeque.
let p1 = w.pop().expect("first pop");
unsafe { p1.execute() };
let p2 = w.pop().expect("second pop");
unsafe { p2.execute() };
let p3 = w.pop().expect("third pop");
unsafe { p3.execute() };
assert!(j1.latch.is_set() && j2.latch.is_set() && j3.latch.is_set());
assert!(w.pop().is_none());
}
#[test]
fn push_always_flushes_for_join_pattern() {
let (w, s) = FclWorker::new(4);
let j = StackJob::new(|_| 7u32, CoreLatch::new());
let r = unsafe { j.as_job_ref(8, 0, Variant::Faithful) };
w.push(r);
// Push auto-flushes; the stealer sees the 1-item slot.
match s.steal() {
chase_lev_local::Steal::Success(slot) => {
assert_eq!(slot.n_items, 1);
unsafe { slot.execute_all_lifo() };
assert!(j.latch.is_set());
}
other => panic!("expected Success, got {other:?}"),
}
}
}