dualcache_ff/
components.rs

1use core::ops::{Deref, DerefMut};
2use crate::sync::atomic::{AtomicUsize, Ordering};
3use crate::sync::index_types::TickType;
4use core::cell::{Cell, RefCell};
5use alloc::vec::Vec;
6
7// ── CachePadded ───────────────────────────────────────────────────────────
8
9/// Cache-line-aligned wrapper to prevent false sharing between worker slots.
10/// Uses `#[repr(C, align(N))]` directly:
11/// - 128 bytes on Apple Silicon / ARM (128-byte cache line)
12/// -  64 bytes on x86_64 and everything else (64-byte cache line)
13#[cfg_attr(any(target_arch = "aarch64", target_arch = "arm"), repr(C, align(128)))]
14#[cfg_attr(not(any(target_arch = "aarch64", target_arch = "arm")), repr(C, align(64)))]
15pub struct CachePadded<T>(pub T);
16
17impl<T> CachePadded<T> {
18    #[inline(always)]
19    pub fn new(val: T) -> Self {
20        Self(val)
21    }
22
23    #[inline(always)]
24    pub fn into_inner(self) -> T {
25        self.0
26    }
27}
28
29impl<T> Deref for CachePadded<T> {
30    type Target = T;
31    #[inline(always)]
32    fn deref(&self) -> &T {
33        &self.0
34    }
35}
36
37impl<T> DerefMut for CachePadded<T> {
38    #[inline(always)]
39    fn deref_mut(&mut self) -> &mut T {
40        &mut self.0
41    }
42}
43
44// ── DefaultSpawner ────────────────────────────────────────────────────────
45
46#[derive(Debug, Clone, Copy, Default)]
47pub struct DefaultSpawner;
48
49impl DefaultSpawner {
50    #[inline]
51    #[cfg(all(feature = "std", not(any(feature = "loom", loom))))]
52    pub fn spawn(&self, f: alloc::boxed::Box<dyn FnOnce() + Send + 'static>) {
53        std::thread::spawn(f);
54    }
55
56    #[inline]
57    #[cfg(any(feature = "loom", loom))]
58    pub fn spawn(&self, f: alloc::boxed::Box<dyn FnOnce() + Send + 'static>) {
59        loom::thread::spawn(move || f());
60    }
61}
62
63// ── DefaultTls ────────────────────────────────────────────────────────────
64
65#[cfg(all(feature = "std", not(any(feature = "loom", loom))))]
66mod std_tls_impl {
67    use super::*;
68    use std::sync::Mutex;
69
70    struct IdAllocator {
71        free_list: Mutex<Vec<usize>>,
72        next_id: AtomicUsize,
73    }
74
75    static ALLOCATOR: IdAllocator = IdAllocator {
76        free_list: Mutex::new(Vec::new()),
77        next_id: AtomicUsize::new(0),
78    };
79
80    struct ThreadIdGuard {
81        id: usize,
82    }
83
84    impl Drop for ThreadIdGuard {
85        fn drop(&mut self) {
86            if let Ok(mut list) = ALLOCATOR.free_list.lock() {
87                list.push(self.id);
88            }
89        }
90    }
91
92    thread_local! {
93        static WORKER_ID: usize = {
94            let id = if let Ok(mut list) = ALLOCATOR.free_list.lock() {
95                list.pop().unwrap_or_else(|| ALLOCATOR.next_id.fetch_add(1, Ordering::Relaxed))
96            } else {
97                ALLOCATOR.next_id.fetch_add(1, Ordering::Relaxed)
98            };
99            
100            GUARD.with(|g| {
101                *g.borrow_mut() = Some(ThreadIdGuard { id });
102            });
103            id
104        };
105
106        static GUARD: RefCell<Option<ThreadIdGuard>> = const { RefCell::new(None) };
107        static HIT_BUF: RefCell<([usize; 64], usize)> = const { RefCell::new(([0; 64], 0)) };
108        static L1_FILTER: RefCell<([u8; 4096], usize)> = const { RefCell::new(([0; 4096], 0)) };
109        static LAST_FLUSH_TICK: Cell<TickType> = const { Cell::new(0) };
110        static WARMUP_STATE: Cell<u8> = const { Cell::new(0) };
111    }
112
113    #[derive(Clone, Default)]
114    pub struct DefaultTls;
115
116    impl DefaultTls {
117        #[inline(always)]
118        pub fn get_worker_id(&self) -> Option<usize> {
119            Some(WORKER_ID.with(|id| *id))
120        }
121
122        #[inline(always)]
123        pub fn with_hit_buf<F, R>(&self, f: F) -> Option<R>
124        where
125            F: FnOnce(&mut ([usize; 64], usize)) -> R,
126        {
127            Some(HIT_BUF.with(|buf| f(&mut buf.borrow_mut())))
128        }
129
130        #[inline(always)]
131        pub fn with_l1_filter<F, R>(&self, f: F) -> Option<R>
132        where
133            F: FnOnce(&mut ([u8; 4096], usize)) -> R,
134        {
135            Some(L1_FILTER.with(|filter| f(&mut filter.borrow_mut())))
136        }
137
138        #[inline(always)]
139        pub fn with_last_flush_tick<F, R>(&self, f: F) -> Option<R>
140        where
141            F: FnOnce(&mut TickType) -> R,
142        {
143            Some(LAST_FLUSH_TICK.with(|cell| {
144                let mut val = cell.get();
145                let res = f(&mut val);
146                cell.set(val);
147                res
148            }))
149        }
150
151        #[inline(always)]
152        pub fn with_warmup_state<F, R>(&self, f: F) -> R
153        where
154            F: FnOnce(&mut u8) -> R,
155        {
156            WARMUP_STATE.with(|cell| {
157                let mut val = cell.get();
158                let res = f(&mut val);
159                cell.set(val);
160                res
161            })
162        }
163    }
164}
165
166#[cfg(all(feature = "std", not(any(feature = "loom", loom))))]
167pub use std_tls_impl::DefaultTls;
168
169#[cfg(any(feature = "loom", loom))]
170mod loom_tls_impl {
171    use super::*;
172    use std::sync::Mutex;
173
174    struct IdAllocator {
175        free_list: Mutex<Vec<usize>>,
176        next_id: AtomicUsize,
177    }
178
179    loom::lazy_static! {
180        static ref ALLOCATOR: IdAllocator = IdAllocator {
181            free_list: Mutex::new(Vec::new()),
182            next_id: AtomicUsize::new(0),
183        };
184    }
185
186    struct ThreadIdGuard {
187        id: usize,
188    }
189
190    impl Drop for ThreadIdGuard {
191        fn drop(&mut self) {
192            if let Ok(mut list) = ALLOCATOR.free_list.lock() {
193                list.push(self.id);
194            }
195        }
196    }
197
198    loom::thread_local! {
199        static WORKER_ID: usize = {
200            let id = if let Ok(mut list) = ALLOCATOR.free_list.lock() {
201                list.pop().unwrap_or_else(|| ALLOCATOR.next_id.fetch_add(1, Ordering::Relaxed))
202            } else {
203                ALLOCATOR.next_id.fetch_add(1, Ordering::Relaxed)
204            };
205            GUARD.with(|g| {
206                *g.borrow_mut() = Some(ThreadIdGuard { id });
207            });
208            id
209        };
210
211        static GUARD: RefCell<Option<ThreadIdGuard>> = RefCell::new(None);
212        static HIT_BUF: RefCell<([usize; 64], usize)> = RefCell::new(([0; 64], 0));
213        static L1_FILTER: RefCell<([u8; 4096], usize)> = RefCell::new(([0; 4096], 0));
214        static LAST_FLUSH_TICK: Cell<TickType> = Cell::new(0);
215        static WARMUP_STATE: Cell<u8> = Cell::new(0);
216    }
217
218    #[derive(Clone, Default)]
219    pub struct DefaultTls;
220
221    impl DefaultTls {
222        #[inline(always)]
223        pub fn get_worker_id(&self) -> Option<usize> {
224            Some(WORKER_ID.with(|id| *id))
225        }
226
227        #[inline(always)]
228        pub fn with_hit_buf<F, R>(&self, f: F) -> Option<R>
229        where
230            F: FnOnce(&mut ([usize; 64], usize)) -> R,
231        {
232            Some(HIT_BUF.with(|buf| f(&mut *buf.borrow_mut())))
233        }
234
235        #[inline(always)]
236        pub fn with_l1_filter<F, R>(&self, f: F) -> Option<R>
237        where
238            F: FnOnce(&mut ([u8; 4096], usize)) -> R,
239        {
240            Some(L1_FILTER.with(|filter| f(&mut *filter.borrow_mut())))
241        }
242
243        #[inline(always)]
244        pub fn with_last_flush_tick<F, R>(&self, f: F) -> Option<R>
245        where
246            F: FnOnce(&mut TickType) -> R,
247        {
248            Some(LAST_FLUSH_TICK.with(|cell| {
249                let mut val = cell.get();
250                let res = f(&mut val);
251                cell.set(val);
252                res
253            }))
254        }
255
256        #[inline(always)]
257        pub fn with_warmup_state<F, R>(&self, f: F) -> R
258        where
259            F: FnOnce(&mut u8) -> R,
260        {
261            WARMUP_STATE.with(|cell| {
262                let mut val = cell.get();
263                let res = f(&mut val);
264                cell.set(val);
265                res
266            })
267        }
268    }
269}
270
271#[cfg(any(feature = "loom", loom))]
272pub use loom_tls_impl::DefaultTls;
273
274#[cfg(not(any(feature = "std", feature = "loom", loom)))]
275mod no_std_tls_impl {
276    use super::*;
277
278    #[derive(Clone, Default)]
279    pub struct DefaultTls;
280
281    impl DefaultTls {
282        #[inline(always)]
283        pub fn get_worker_id(&self) -> Option<usize> {
284            None
285        }
286
287        #[inline(always)]
288        pub fn with_hit_buf<F, R>(&self, _f: F) -> Option<R>
289        where
290            F: FnOnce(&mut ([usize; 64], usize)) -> R,
291        {
292            None
293        }
294
295        #[inline(always)]
296        pub fn with_l1_filter<F, R>(&self, _f: F) -> Option<R>
297        where
298            F: FnOnce(&mut ([u8; 4096], usize)) -> R,
299        {
300            None
301        }
302
303        #[inline(always)]
304        pub fn with_last_flush_tick<F, R>(&self, _f: F) -> Option<R>
305        where
306            F: FnOnce(&mut TickType) -> R,
307        {
308            None
309        }
310
311        #[inline(always)]
312        pub fn with_warmup_state<F, R>(&self, f: F) -> R
313        where
314            F: FnOnce(&mut u8) -> R,
315        {
316            let mut state = 255;
317            f(&mut state)
318        }
319    }
320}
321
322#[cfg(not(any(feature = "std", feature = "loom", loom)))]
323pub use no_std_tls_impl::DefaultTls;
dualcache_ff/components.rs

dualcache_ff/
components.rs
