1use std::{
2 cell::RefCell,
3 convert::Infallible,
4 fmt,
5 sync::{
6 Arc, Condvar, Mutex,
7 atomic::{AtomicBool, Ordering},
8 },
9 time::{Duration, Instant},
10};
11
12use crossbeam_channel::{Receiver, Sender, bounded};
13use rusttorch_core::{Result, RustTorchError};
14
15pub const WORKER_SEED_DERIVATION_VERSION: u32 = 1;
17
18struct CancellationState {
24 cancelled: AtomicBool,
25 wait_lock: Mutex<()>,
26 deadline: Mutex<Option<Instant>>,
27 wake: Condvar,
28 signal: Receiver<()>,
29 signal_sender: Mutex<Option<Sender<()>>>,
30 #[cfg(test)]
31 require_wait_lock_for_transition: AtomicBool,
32}
33
34impl CancellationState {
35 fn new(deadline: Option<Instant>) -> Arc<Self> {
36 let (signal_sender, signal) = bounded(0);
37 Arc::new(Self {
38 cancelled: AtomicBool::new(false),
39 wait_lock: Mutex::new(()),
40 deadline: Mutex::new(deadline),
41 wake: Condvar::new(),
42 signal,
43 signal_sender: Mutex::new(Some(signal_sender)),
44 #[cfg(test)]
45 require_wait_lock_for_transition: AtomicBool::new(false),
46 })
47 }
48
49 #[cfg(test)]
50 fn assert_transition_holds_wait_lock(&self) {
51 if self.require_wait_lock_for_transition.load(Ordering::SeqCst) {
52 assert!(
53 self.wait_lock.try_lock().is_err(),
54 "predicate transition did not hold the waiter mutex"
55 );
56 }
57 }
58}
59
60#[derive(Clone)]
65pub struct CancellationToken {
66 state: Arc<CancellationState>,
67}
68
69impl fmt::Debug for CancellationToken {
70 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
71 formatter
72 .debug_struct("CancellationToken")
73 .field("cancelled", &self.is_cancelled())
74 .finish_non_exhaustive()
75 }
76}
77
78impl Default for CancellationToken {
79 fn default() -> Self {
80 Self::new()
81 }
82}
83
84impl CancellationToken {
85 pub fn new() -> Self {
87 Self {
88 state: CancellationState::new(None),
89 }
90 }
91
92 pub fn cancel(&self) {
94 let _wait_guard = self.state.wait_lock.lock().unwrap();
95 #[cfg(test)]
96 self.state.assert_transition_holds_wait_lock();
97 if !self.state.cancelled.swap(true, Ordering::AcqRel) {
98 self.state.signal_sender.lock().unwrap().take();
99 self.state.wake.notify_all();
100 }
101 }
102
103 pub fn is_cancelled(&self) -> bool {
105 self.state.cancelled.load(Ordering::Acquire)
106 }
107
108 pub fn wait_cancelled(&self) {
110 let mut guard = self.state.wait_lock.lock().unwrap();
111 while !self.is_cancelled() {
112 guard = self.state.wake.wait(guard).unwrap();
113 }
114 }
115
116 pub fn wait_cancelled_timeout(&self, timeout: Duration) -> bool {
118 if self.is_cancelled() {
119 return true;
120 }
121 let guard = self.state.wait_lock.lock().unwrap();
122 let _ = self
123 .state
124 .wake
125 .wait_timeout_while(guard, timeout, |_| !self.is_cancelled())
126 .unwrap();
127 self.is_cancelled()
128 }
129
130 pub(crate) fn signal(&self) -> &Receiver<()> {
131 &self.state.signal
132 }
133
134 pub(crate) fn paired_deadline(&self) -> Deadline {
135 Deadline {
136 state: Arc::clone(&self.state),
137 }
138 }
139
140 pub(crate) fn wait_cancelled_or_deadline(&self, deadline: &Deadline) -> WaitOutcome {
141 let mut guard = self.state.wait_lock.lock().unwrap();
142 loop {
143 if self.is_cancelled() {
144 return WaitOutcome::Cancelled;
145 }
146 match deadline.remaining() {
147 Some(remaining) if remaining.is_zero() => return WaitOutcome::DeadlineExpired,
148 Some(remaining) => {
149 let (next, _) = self.state.wake.wait_timeout(guard, remaining).unwrap();
150 guard = next;
151 }
152 None => guard = self.state.wake.wait(guard).unwrap(),
153 }
154 }
155 }
156}
157
158#[derive(Clone)]
163pub struct Deadline {
164 state: Arc<CancellationState>,
165}
166
167impl fmt::Debug for Deadline {
168 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
169 formatter
170 .debug_struct("Deadline")
171 .field("remaining", &self.remaining())
172 .finish_non_exhaustive()
173 }
174}
175
176impl Default for Deadline {
177 fn default() -> Self {
178 Self::none()
179 }
180}
181
182impl Deadline {
183 pub fn none() -> Self {
185 Self {
186 state: CancellationState::new(None),
187 }
188 }
189
190 pub fn after(timeout: Duration) -> Self {
195 let now = Instant::now();
196 Self {
197 state: CancellationState::new(Some(now.checked_add(timeout).unwrap_or(now))),
198 }
199 }
200
201 pub fn is_expired(&self) -> bool {
203 self.remaining()
204 .is_some_and(|remaining| remaining.is_zero())
205 }
206
207 pub fn remaining(&self) -> Option<Duration> {
209 self.state
210 .deadline
211 .lock()
212 .unwrap()
213 .map(|deadline| deadline.saturating_duration_since(Instant::now()))
214 }
215
216 pub(crate) fn arm(&self, timeout: Duration) {
217 let _wait_guard = self.state.wait_lock.lock().unwrap();
218 #[cfg(test)]
219 self.state.assert_transition_holds_wait_lock();
220 let now = Instant::now();
221 *self.state.deadline.lock().unwrap() = Some(now.checked_add(timeout).unwrap_or(now));
222 self.state.wake.notify_all();
223 }
224
225 pub(crate) fn disarm(&self) {
226 let _wait_guard = self.state.wait_lock.lock().unwrap();
227 #[cfg(test)]
228 self.state.assert_transition_holds_wait_lock();
229 *self.state.deadline.lock().unwrap() = None;
230 self.state.wake.notify_all();
231 }
232
233 pub(crate) fn can_represent(timeout: Duration) -> bool {
234 Instant::now().checked_add(timeout).is_some()
235 }
236}
237
238#[derive(Clone, Copy, Debug, Eq, PartialEq)]
240pub enum WaitOutcome {
241 Cancelled,
243 DeadlineExpired,
245}
246
247#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)]
249#[error("loader work was cancelled or its deadline expired")]
250pub struct LoaderCancelled;
251
252#[derive(Clone, Copy, Debug, Eq, PartialEq)]
265pub struct WorkerInfo {
266 pub id: usize,
268 pub num_workers: usize,
270 pub seed: u64,
272 pub rank: usize,
274}
275
276impl WorkerInfo {
277 pub fn new(id: usize, num_workers: usize, seed: u64, rank: usize) -> Result<Self> {
284 validate_worker(id, num_workers)?;
285 Ok(Self {
286 id,
287 num_workers,
288 seed,
289 rank,
290 })
291 }
292
293 pub fn from_loader_seed(
304 id: usize,
305 num_workers: usize,
306 loader_seed: u64,
307 rank: usize,
308 iterator_generation: u64,
309 ) -> Result<Self> {
310 Self::new(
311 id,
312 num_workers,
313 derive_worker_seed(loader_seed, rank, iterator_generation, id),
314 rank,
315 )
316 }
317}
318
319#[derive(Clone, Debug)]
357pub struct WorkerContext {
358 pub info: WorkerInfo,
360 pub cancellation: CancellationToken,
362 pub deadline: Deadline,
364}
365
366impl WorkerContext {
367 pub fn new(info: WorkerInfo, cancellation: CancellationToken, deadline: Deadline) -> Self {
369 Self {
370 info,
371 cancellation,
372 deadline,
373 }
374 }
375
376 pub fn check(&self) -> std::result::Result<(), LoaderCancelled> {
378 if self.cancellation.is_cancelled() || self.deadline.is_expired() {
379 Err(LoaderCancelled)
380 } else {
381 Ok(())
382 }
383 }
384
385 pub fn wait_cancelled_or_deadline(&self) -> WaitOutcome {
387 self.cancellation.wait_cancelled_or_deadline(&self.deadline)
388 }
389}
390
391pub trait WorkerInit {
396 type Error;
398
399 fn initialize(&self, worker: &WorkerContext) -> std::result::Result<(), Self::Error>;
401}
402
403#[derive(Clone, Copy, Debug)]
405pub struct FnWorkerInit<F> {
406 initialize: F,
407}
408
409impl<F> FnWorkerInit<F> {
410 pub fn new(initialize: F) -> Self {
412 Self { initialize }
413 }
414}
415
416impl<E, F> WorkerInit for FnWorkerInit<F>
417where
418 F: Fn(&WorkerContext) -> std::result::Result<(), E>,
419{
420 type Error = E;
421
422 fn initialize(&self, worker: &WorkerContext) -> std::result::Result<(), Self::Error> {
423 (self.initialize)(worker)
424 }
425}
426
427#[derive(Clone, Copy, Debug, Default)]
429pub struct NoWorkerInit;
430
431impl WorkerInit for NoWorkerInit {
432 type Error = Infallible;
433
434 fn initialize(&self, _worker: &WorkerContext) -> std::result::Result<(), Self::Error> {
435 Ok(())
436 }
437}
438
439thread_local! {
440 static WORKER_INFO: RefCell<Option<WorkerInfo>> = const { RefCell::new(None) };
441}
442
443pub fn get_worker_info() -> Option<WorkerInfo> {
451 WORKER_INFO.with(|current| *current.borrow())
452}
453
454#[doc(hidden)]
459pub fn with_worker_info<T>(worker: WorkerInfo, callback: impl FnOnce() -> T) -> T {
460 struct Restore(Option<WorkerInfo>);
461
462 impl Drop for Restore {
463 fn drop(&mut self) {
464 WORKER_INFO.with(|current| *current.borrow_mut() = self.0);
465 }
466 }
467
468 let previous = WORKER_INFO.with(|current| current.borrow_mut().replace(worker));
469 let _restore = Restore(previous);
470 callback()
471}
472
473fn validate_worker(id: usize, num_workers: usize) -> Result<()> {
474 if num_workers == 0 {
475 return Err(invalid_configuration(
476 "num_workers",
477 "must be greater than zero",
478 ));
479 }
480 if id >= num_workers {
481 return Err(invalid_configuration(
482 "worker_id",
483 "must be less than num_workers",
484 ));
485 }
486 Ok(())
487}
488
489fn splitmix64(mut value: u64) -> u64 {
490 value = value.wrapping_add(0x9e37_79b9_7f4a_7c15);
491 value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
492 value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
493 value ^ (value >> 31)
494}
495
496fn derive_worker_seed(
497 loader_seed: u64,
498 rank: usize,
499 iterator_generation: u64,
500 worker_id: usize,
501) -> u64 {
502 let rank_domain = (rank as u64).wrapping_mul(0xd1b5_4a32_d192_ed03);
503 let iterator_domain = iterator_generation.wrapping_mul(0xa076_1d64_78bd_642f);
504 splitmix64(loader_seed ^ rank_domain ^ iterator_domain).wrapping_add(worker_id as u64)
505}
506
507fn invalid_configuration(field: &'static str, reason: impl Into<String>) -> RustTorchError {
508 RustTorchError::InvalidConfiguration {
509 field,
510 reason: reason.into(),
511 }
512}
513
514#[cfg(test)]
515mod tests {
516 use super::*;
517
518 #[test]
519 fn cancellation_transition_holds_the_waiter_mutex() {
520 let cancellation = CancellationToken::new();
521 cancellation
522 .state
523 .require_wait_lock_for_transition
524 .store(true, Ordering::SeqCst);
525
526 cancellation.cancel();
527 assert!(cancellation.is_cancelled());
528 }
529
530 #[test]
531 fn deadline_transitions_hold_the_waiter_mutex() {
532 let cancellation = CancellationToken::new();
533 let deadline = cancellation.paired_deadline();
534 cancellation
535 .state
536 .require_wait_lock_for_transition
537 .store(true, Ordering::SeqCst);
538
539 deadline.arm(Duration::from_secs(1));
540 deadline.disarm();
541 assert_eq!(deadline.remaining(), None);
542 }
543}