Skip to main content

grommet_offload/
lib.rs

1//! Pinned, bounded Rayon pools for CPU-bound work submitted from shards.
2//!
3//! Shard cores and compute cores are disjoint on purpose. A shard core spends
4//! its time dispatching and awaiting; a compute core spends its time saturated.
5//! Running both on one core makes each worse, and the split is what keeps a long
6//! computation from adding latency to unrelated keys.
7//!
8//! There is a pool per memory node rather than one pool for the machine. A shard
9//! that ships a closure and its captured data to a worker on another node pays
10//! the interconnect on every task, which is exactly the cost that pinning the
11//! shard was meant to avoid, so [`OffloadPools`] builds the set a [`Plan`] asks
12//! for, and each shard submits to the one local to itself.
13
14#![deny(unsafe_code)]
15
16use grommet::offload::{Offload, OffloadError};
17use grommet::topology::{OffloadPool, Plan};
18use std::sync::Arc;
19use std::sync::atomic::{AtomicU64, Ordering::Relaxed};
20use std::time::{Duration, Instant};
21
22#[derive(Debug, Default)]
23pub struct OffloadStats {
24    pub runs: AtomicU64,
25    pub panics: AtomicU64,
26    pub latency_nanos_sum: AtomicU64,
27    pub latency_max_nanos: AtomicU64,
28    /// Time spent waiting for a permit, which is what backpressure from a
29    /// saturated pool looks like from the shard's side. This is the number
30    /// [`calibrate`] reads to decide the pool is too small.
31    ///
32    /// [`calibrate`]: grommet::topology::calibrate
33    pub permit_wait_nanos: AtomicU64,
34    /// Workers whose CPU binding took effect. Fewer than [`RayonOffload::workers`]
35    /// means some of this pool is running wherever the OS put it.
36    pub bound_workers: AtomicU64,
37}
38
39impl OffloadStats {
40    fn record(&self, elapsed: Duration) {
41        let nanos = elapsed.as_nanos() as u64;
42        self.runs.fetch_add(1, Relaxed);
43        self.latency_nanos_sum.fetch_add(nanos, Relaxed);
44        self.latency_max_nanos.fetch_max(nanos, Relaxed);
45    }
46
47    /// Mean completed-task latency in nanoseconds, or zero before the first task
48    /// completes.
49    pub fn mean_latency_nanos(&self) -> u64 {
50        let runs = self.runs.load(Relaxed);
51        if runs == 0 {
52            return 0;
53        }
54        self.latency_nanos_sum.load(Relaxed) / runs
55    }
56
57    /// Summed duration of completed tasks. Workers are saturated for the whole
58    /// of a task, so over a window this is compute demand in core-seconds.
59    pub fn busy(&self) -> Duration {
60        Duration::from_nanos(self.latency_nanos_sum.load(Relaxed))
61    }
62
63    pub fn permit_wait(&self) -> Duration {
64        Duration::from_nanos(self.permit_wait_nanos.load(Relaxed))
65    }
66}
67
68/// A Rayon pool whose workers are bound to dedicated CPUs on one memory node,
69/// with a bound on how many tasks may be outstanding at once.
70#[derive(Clone)]
71pub struct RayonOffload {
72    pool: Arc<rayon::ThreadPool>,
73    permits: Arc<tokio::sync::Semaphore>,
74    stats: Arc<OffloadStats>,
75    workers: usize,
76    node: Option<usize>,
77}
78
79impl RayonOffload {
80    /// One worker per CPU in `pool`, bound to it, with four outstanding tasks
81    /// per worker.
82    pub fn for_pool(plan: Arc<Plan>, pool: &OffloadPool) -> Self {
83        Self::with_queue_depth(plan, pool, 4)
84    }
85
86    /// `queue_depth` outstanding tasks per worker. Deeper keeps workers fed
87    /// through bursts; shallower makes a shard feel backpressure sooner, which
88    /// is usually what a latency objective wants, and makes
89    /// `permit_wait_nanos` a more responsive signal that the pool is too small.
90    pub fn with_queue_depth(plan: Arc<Plan>, pool: &OffloadPool, queue_depth: usize) -> Self {
91        let workers = pool.cpus.len().max(1);
92        let node = pool.node;
93        let placement = pool.clone();
94        let stats = Arc::new(OffloadStats::default());
95        let binding = stats.clone();
96        Self::assemble(
97            workers,
98            queue_depth,
99            Some(node),
100            stats,
101            Some(Arc::new(move |worker: usize| {
102                if plan.bind_offload_worker(&placement, worker).cpu {
103                    binding.bound_workers.fetch_add(1, Relaxed);
104                }
105            })),
106        )
107    }
108
109    /// A pool of `workers` threads that are not bound to anything.
110    ///
111    /// For tests, for platforms that will not honour binding, and for callers
112    /// with no plan. It is a working pool, not a placed one: the OS decides
113    /// where these threads run, including on top of the shard reactors.
114    pub fn unbound(workers: usize, queue_depth: usize) -> Self {
115        Self::assemble(workers.max(1), queue_depth, None, Arc::new(OffloadStats::default()), None)
116    }
117
118    fn assemble(
119        workers: usize,
120        queue_depth: usize,
121        node: Option<usize>,
122        stats: Arc<OffloadStats>,
123        bind: Option<Arc<dyn Fn(usize) + Send + Sync>>,
124    ) -> Self {
125        assert!(queue_depth > 0, "a compute pool needs a queue depth");
126        let panic_stats = stats.clone();
127        let pool = rayon::ThreadPoolBuilder::new()
128            .num_threads(workers)
129            .thread_name(move |index| match node {
130                Some(node) => format!("offload-n{node}-{index}"),
131                None => format!("offload-{index}"),
132            })
133            .start_handler(move |index| {
134                if let Some(bind) = &bind {
135                    bind(index);
136                }
137            })
138            // Without a handler, Rayon aborts the process when a spawned task
139            // panics. A panicking task must only fail its own caller, which it
140            // does: the dropped responder surfaces as `WorkerLost`.
141            .panic_handler(move |_| {
142                panic_stats.panics.fetch_add(1, Relaxed);
143            })
144            .build()
145            .expect("build offload pool");
146
147        Self {
148            pool: Arc::new(pool),
149            permits: Arc::new(tokio::sync::Semaphore::new(workers * queue_depth)),
150            stats,
151            workers,
152            node,
153        }
154    }
155
156    pub fn workers(&self) -> usize {
157        self.workers
158    }
159
160    /// The memory node this pool's workers sit on, if it was placed.
161    pub fn node(&self) -> Option<usize> {
162        self.node
163    }
164
165    pub fn stats(&self) -> &Arc<OffloadStats> {
166        &self.stats
167    }
168}
169
170/// The pools a [`Plan`] calls for: one per memory node, each bound to its own.
171#[derive(Clone, Default)]
172pub struct OffloadPools {
173    pools: Vec<RayonOffload>,
174}
175
176impl OffloadPools {
177    /// Build every pool in `plan`. Empty when the plan reserved no compute
178    /// cores, which is what a `compute_fraction` of zero asks for.
179    pub fn build(plan: Arc<Plan>, queue_depth: usize) -> Self {
180        let pools = plan
181            .offload
182            .iter()
183            .map(|pool| RayonOffload::with_queue_depth(plan.clone(), pool, queue_depth))
184            .collect();
185        Self { pools }
186    }
187
188    /// The pool local to `node`, falling back to the first: a shard is better
189    /// served by a remote pool than by no pool.
190    pub fn for_node(&self, node: Option<usize>) -> Option<RayonOffload> {
191        node.and_then(|node| self.pools.iter().find(|pool| pool.node == Some(node)))
192            .or_else(|| self.pools.first())
193            .cloned()
194    }
195
196    pub fn is_empty(&self) -> bool {
197        self.pools.is_empty()
198    }
199
200    pub fn workers(&self) -> usize {
201        self.pools.iter().map(RayonOffload::workers).sum()
202    }
203
204    /// Workers across every pool whose binding took effect.
205    pub fn bound_workers(&self) -> usize {
206        self.pools.iter().map(|pool| pool.stats().bound_workers.load(Relaxed) as usize).sum()
207    }
208
209    pub fn stats(&self) -> Vec<Arc<OffloadStats>> {
210        self.pools.iter().map(|pool| pool.stats().clone()).collect()
211    }
212}
213
214impl Offload for RayonOffload {
215    async fn run<F, T>(&self, task: F) -> Result<T, OffloadError>
216    where
217        F: FnOnce() -> T + Send + 'static,
218        T: Send + 'static,
219    {
220        let waiting = Instant::now();
221        let _permit =
222            self.permits.clone().acquire_owned().await.map_err(|_| OffloadError::Closed)?;
223        self.stats.permit_wait_nanos.fetch_add(waiting.elapsed().as_nanos() as u64, Relaxed);
224
225        let (respond, receive) = tokio::sync::oneshot::channel();
226        let started = Instant::now();
227        self.pool.spawn_fifo(move || {
228            let _ = respond.send(task());
229        });
230        let output = receive.await.map_err(|_| OffloadError::WorkerLost)?;
231        self.stats.record(started.elapsed());
232        Ok(output)
233    }
234}
235
236#[cfg(test)]
237mod tests {
238    use super::*;
239    use grommet::topology::{Workload, detect};
240
241    fn pool() -> RayonOffload {
242        RayonOffload::unbound(2, 2)
243    }
244
245    #[tokio::test]
246    async fn work_runs_off_the_calling_thread_and_returns_its_value() {
247        let offload = pool();
248        let caller = std::thread::current().id();
249        let worker = offload.run(move || (6 * 7, std::thread::current().id())).await.unwrap();
250        assert_eq!(worker.0, 42);
251        assert_ne!(worker.1, caller, "compute must not run on the shard core");
252        assert_eq!(offload.stats().runs.load(Relaxed), 1);
253        assert!(offload.stats().mean_latency_nanos() > 0);
254        assert!(offload.stats().busy() > Duration::ZERO);
255    }
256
257    #[tokio::test]
258    async fn a_panicking_task_fails_only_its_caller() {
259        let offload = pool();
260        let failed: Result<(), _> = offload.run(|| panic!("compute blew up")).await;
261        assert_eq!(failed, Err(OffloadError::WorkerLost));
262
263        // The caller learns of the failure when the responder is dropped during
264        // unwinding, which happens before Rayon reaches its panic handler, so
265        // the counter is not readable the instant `run` returns.
266        for _ in 0..1_000 {
267            if offload.stats().panics.load(Relaxed) == 1 {
268                break;
269            }
270            tokio::time::sleep(Duration::from_millis(1)).await;
271        }
272        assert_eq!(offload.stats().panics.load(Relaxed), 1);
273
274        // The pool is still usable, which is the point of catching it.
275        assert_eq!(offload.run(|| 1 + 1).await, Ok(2));
276    }
277
278    #[tokio::test]
279    async fn many_concurrent_tasks_all_complete_under_a_bounded_permit_count() {
280        let offload = pool();
281        let tasks = (0..64u64).map(|value| {
282            let offload = offload.clone();
283            tokio::spawn(async move { offload.run(move || value * 2).await })
284        });
285        let mut total = 0;
286        for task in tasks {
287            total += task.await.unwrap().unwrap();
288        }
289        assert_eq!(total, (0..64u64).map(|value| value * 2).sum::<u64>());
290        assert_eq!(offload.stats().runs.load(Relaxed), 64);
291    }
292
293    #[test]
294    fn mean_latency_is_zero_before_anything_runs() {
295        assert_eq!(OffloadStats::default().mean_latency_nanos(), 0);
296        assert_eq!(OffloadStats::default().busy(), Duration::ZERO);
297    }
298
299    #[tokio::test]
300    async fn a_plan_produces_one_working_pool_per_memory_node() {
301        let plan = Arc::new(detect(&Workload::default()).expect("read this machine"));
302        let pools = OffloadPools::build(plan.clone(), 2);
303        assert_eq!(pools.workers(), plan.offload_workers());
304
305        for shard in &plan.shards {
306            let Some(offload) = pools.for_node(Some(shard.node)) else {
307                assert!(pools.is_empty(), "a non-empty set must serve every shard");
308                continue;
309            };
310            // A shard's pool is either on its own node or the fallback, never on
311            // some third node picked at random.
312            assert!(
313                offload.node() == Some(shard.node) || offload.node() == pools.pools[0].node,
314                "shard on node {} got a pool on {:?}",
315                shard.node,
316                offload.node(),
317            );
318            assert_eq!(offload.run(|| 6 * 7).await, Ok(42));
319        }
320    }
321
322    #[test]
323    fn an_unplaced_pool_is_still_a_pool() {
324        let offload = RayonOffload::unbound(0, 1);
325        assert_eq!(offload.workers(), 1, "zero workers would be a pool that cannot run anything");
326        assert_eq!(offload.node(), None);
327        assert_eq!(offload.stats().bound_workers.load(Relaxed), 0);
328    }
329
330    #[test]
331    fn an_empty_set_of_pools_serves_nobody_rather_than_pretending() {
332        // `compute_fraction: 0.0` plans no offload cores. Handing back a silent
333        // inline pool would put compute back on the reactors it was moved off.
334        let plan = Arc::new(
335            detect(&Workload { compute_fraction: 0.0, ..Workload::default() })
336                .expect("read this machine"),
337        );
338        let pools = OffloadPools::build(plan, 2);
339        assert!(pools.is_empty());
340        assert_eq!(pools.workers(), 0);
341        assert!(pools.for_node(Some(0)).is_none());
342    }
343}