1#![deny(unsafe_code)]
15
16use grommet::offload::{Offload, OffloadError};
17use grommet::topology::{OffloadPool, Plan};
18use std::sync::Arc;
19use std::sync::atomic::{AtomicU64, Ordering::Relaxed};
20use std::time::{Duration, Instant};
21
22#[derive(Debug, Default)]
23pub struct OffloadStats {
24 pub runs: AtomicU64,
25 pub panics: AtomicU64,
26 pub latency_nanos_sum: AtomicU64,
27 pub latency_max_nanos: AtomicU64,
28 pub permit_wait_nanos: AtomicU64,
34 pub bound_workers: AtomicU64,
37}
38
39impl OffloadStats {
40 fn record(&self, elapsed: Duration) {
41 let nanos = elapsed.as_nanos() as u64;
42 self.runs.fetch_add(1, Relaxed);
43 self.latency_nanos_sum.fetch_add(nanos, Relaxed);
44 self.latency_max_nanos.fetch_max(nanos, Relaxed);
45 }
46
47 pub fn mean_latency_nanos(&self) -> u64 {
50 let runs = self.runs.load(Relaxed);
51 if runs == 0 {
52 return 0;
53 }
54 self.latency_nanos_sum.load(Relaxed) / runs
55 }
56
57 pub fn busy(&self) -> Duration {
60 Duration::from_nanos(self.latency_nanos_sum.load(Relaxed))
61 }
62
63 pub fn permit_wait(&self) -> Duration {
64 Duration::from_nanos(self.permit_wait_nanos.load(Relaxed))
65 }
66}
67
68#[derive(Clone)]
71pub struct RayonOffload {
72 pool: Arc<rayon::ThreadPool>,
73 permits: Arc<tokio::sync::Semaphore>,
74 stats: Arc<OffloadStats>,
75 workers: usize,
76 node: Option<usize>,
77}
78
79impl RayonOffload {
80 pub fn for_pool(plan: Arc<Plan>, pool: &OffloadPool) -> Self {
83 Self::with_queue_depth(plan, pool, 4)
84 }
85
86 pub fn with_queue_depth(plan: Arc<Plan>, pool: &OffloadPool, queue_depth: usize) -> Self {
91 let workers = pool.cpus.len().max(1);
92 let node = pool.node;
93 let placement = pool.clone();
94 let stats = Arc::new(OffloadStats::default());
95 let binding = stats.clone();
96 Self::assemble(
97 workers,
98 queue_depth,
99 Some(node),
100 stats,
101 Some(Arc::new(move |worker: usize| {
102 if plan.bind_offload_worker(&placement, worker).cpu {
103 binding.bound_workers.fetch_add(1, Relaxed);
104 }
105 })),
106 )
107 }
108
109 pub fn unbound(workers: usize, queue_depth: usize) -> Self {
115 Self::assemble(workers.max(1), queue_depth, None, Arc::new(OffloadStats::default()), None)
116 }
117
118 fn assemble(
119 workers: usize,
120 queue_depth: usize,
121 node: Option<usize>,
122 stats: Arc<OffloadStats>,
123 bind: Option<Arc<dyn Fn(usize) + Send + Sync>>,
124 ) -> Self {
125 assert!(queue_depth > 0, "a compute pool needs a queue depth");
126 let panic_stats = stats.clone();
127 let pool = rayon::ThreadPoolBuilder::new()
128 .num_threads(workers)
129 .thread_name(move |index| match node {
130 Some(node) => format!("offload-n{node}-{index}"),
131 None => format!("offload-{index}"),
132 })
133 .start_handler(move |index| {
134 if let Some(bind) = &bind {
135 bind(index);
136 }
137 })
138 .panic_handler(move |_| {
142 panic_stats.panics.fetch_add(1, Relaxed);
143 })
144 .build()
145 .expect("build offload pool");
146
147 Self {
148 pool: Arc::new(pool),
149 permits: Arc::new(tokio::sync::Semaphore::new(workers * queue_depth)),
150 stats,
151 workers,
152 node,
153 }
154 }
155
156 pub fn workers(&self) -> usize {
157 self.workers
158 }
159
160 pub fn node(&self) -> Option<usize> {
162 self.node
163 }
164
165 pub fn stats(&self) -> &Arc<OffloadStats> {
166 &self.stats
167 }
168}
169
170#[derive(Clone, Default)]
172pub struct OffloadPools {
173 pools: Vec<RayonOffload>,
174}
175
176impl OffloadPools {
177 pub fn build(plan: Arc<Plan>, queue_depth: usize) -> Self {
180 let pools = plan
181 .offload
182 .iter()
183 .map(|pool| RayonOffload::with_queue_depth(plan.clone(), pool, queue_depth))
184 .collect();
185 Self { pools }
186 }
187
188 pub fn for_node(&self, node: Option<usize>) -> Option<RayonOffload> {
191 node.and_then(|node| self.pools.iter().find(|pool| pool.node == Some(node)))
192 .or_else(|| self.pools.first())
193 .cloned()
194 }
195
196 pub fn is_empty(&self) -> bool {
197 self.pools.is_empty()
198 }
199
200 pub fn workers(&self) -> usize {
201 self.pools.iter().map(RayonOffload::workers).sum()
202 }
203
204 pub fn bound_workers(&self) -> usize {
206 self.pools.iter().map(|pool| pool.stats().bound_workers.load(Relaxed) as usize).sum()
207 }
208
209 pub fn stats(&self) -> Vec<Arc<OffloadStats>> {
210 self.pools.iter().map(|pool| pool.stats().clone()).collect()
211 }
212}
213
214impl Offload for RayonOffload {
215 async fn run<F, T>(&self, task: F) -> Result<T, OffloadError>
216 where
217 F: FnOnce() -> T + Send + 'static,
218 T: Send + 'static,
219 {
220 let waiting = Instant::now();
221 let _permit =
222 self.permits.clone().acquire_owned().await.map_err(|_| OffloadError::Closed)?;
223 self.stats.permit_wait_nanos.fetch_add(waiting.elapsed().as_nanos() as u64, Relaxed);
224
225 let (respond, receive) = tokio::sync::oneshot::channel();
226 let started = Instant::now();
227 self.pool.spawn_fifo(move || {
228 let _ = respond.send(task());
229 });
230 let output = receive.await.map_err(|_| OffloadError::WorkerLost)?;
231 self.stats.record(started.elapsed());
232 Ok(output)
233 }
234}
235
236#[cfg(test)]
237mod tests {
238 use super::*;
239 use grommet::topology::{Workload, detect};
240
241 fn pool() -> RayonOffload {
242 RayonOffload::unbound(2, 2)
243 }
244
245 #[tokio::test]
246 async fn work_runs_off_the_calling_thread_and_returns_its_value() {
247 let offload = pool();
248 let caller = std::thread::current().id();
249 let worker = offload.run(move || (6 * 7, std::thread::current().id())).await.unwrap();
250 assert_eq!(worker.0, 42);
251 assert_ne!(worker.1, caller, "compute must not run on the shard core");
252 assert_eq!(offload.stats().runs.load(Relaxed), 1);
253 assert!(offload.stats().mean_latency_nanos() > 0);
254 assert!(offload.stats().busy() > Duration::ZERO);
255 }
256
257 #[tokio::test]
258 async fn a_panicking_task_fails_only_its_caller() {
259 let offload = pool();
260 let failed: Result<(), _> = offload.run(|| panic!("compute blew up")).await;
261 assert_eq!(failed, Err(OffloadError::WorkerLost));
262
263 for _ in 0..1_000 {
267 if offload.stats().panics.load(Relaxed) == 1 {
268 break;
269 }
270 tokio::time::sleep(Duration::from_millis(1)).await;
271 }
272 assert_eq!(offload.stats().panics.load(Relaxed), 1);
273
274 assert_eq!(offload.run(|| 1 + 1).await, Ok(2));
276 }
277
278 #[tokio::test]
279 async fn many_concurrent_tasks_all_complete_under_a_bounded_permit_count() {
280 let offload = pool();
281 let tasks = (0..64u64).map(|value| {
282 let offload = offload.clone();
283 tokio::spawn(async move { offload.run(move || value * 2).await })
284 });
285 let mut total = 0;
286 for task in tasks {
287 total += task.await.unwrap().unwrap();
288 }
289 assert_eq!(total, (0..64u64).map(|value| value * 2).sum::<u64>());
290 assert_eq!(offload.stats().runs.load(Relaxed), 64);
291 }
292
293 #[test]
294 fn mean_latency_is_zero_before_anything_runs() {
295 assert_eq!(OffloadStats::default().mean_latency_nanos(), 0);
296 assert_eq!(OffloadStats::default().busy(), Duration::ZERO);
297 }
298
299 #[tokio::test]
300 async fn a_plan_produces_one_working_pool_per_memory_node() {
301 let plan = Arc::new(detect(&Workload::default()).expect("read this machine"));
302 let pools = OffloadPools::build(plan.clone(), 2);
303 assert_eq!(pools.workers(), plan.offload_workers());
304
305 for shard in &plan.shards {
306 let Some(offload) = pools.for_node(Some(shard.node)) else {
307 assert!(pools.is_empty(), "a non-empty set must serve every shard");
308 continue;
309 };
310 assert!(
313 offload.node() == Some(shard.node) || offload.node() == pools.pools[0].node,
314 "shard on node {} got a pool on {:?}",
315 shard.node,
316 offload.node(),
317 );
318 assert_eq!(offload.run(|| 6 * 7).await, Ok(42));
319 }
320 }
321
322 #[test]
323 fn an_unplaced_pool_is_still_a_pool() {
324 let offload = RayonOffload::unbound(0, 1);
325 assert_eq!(offload.workers(), 1, "zero workers would be a pool that cannot run anything");
326 assert_eq!(offload.node(), None);
327 assert_eq!(offload.stats().bound_workers.load(Relaxed), 0);
328 }
329
330 #[test]
331 fn an_empty_set_of_pools_serves_nobody_rather_than_pretending() {
332 let plan = Arc::new(
335 detect(&Workload { compute_fraction: 0.0, ..Workload::default() })
336 .expect("read this machine"),
337 );
338 let pools = OffloadPools::build(plan, 2);
339 assert!(pools.is_empty());
340 assert_eq!(pools.workers(), 0);
341 assert!(pools.for_node(Some(0)).is_none());
342 }
343}