1use std::{
2 cell::RefCell,
3 convert::Infallible,
4 fmt,
5 sync::{
6 Arc, Condvar, Mutex,
7 atomic::{AtomicBool, Ordering},
8 },
9 time::{Duration, Instant},
10};
11
12use crossbeam_channel::{Receiver, Sender, bounded};
13use rusttorch_core::{Result, RustTorchError};
14
15pub const WORKER_SEED_DERIVATION_VERSION: u32 = 1;
17
18struct CancellationState {
24 cancelled: AtomicBool,
25 wait_lock: Mutex<()>,
26 deadline: Mutex<Option<Instant>>,
27 wake: Condvar,
28 signal: Receiver<()>,
29 signal_sender: Mutex<Option<Sender<()>>>,
30 #[cfg(test)]
31 require_wait_lock_for_transition: AtomicBool,
32}
33
34impl CancellationState {
35 fn new(deadline: Option<Instant>) -> Arc<Self> {
36 let (signal_sender, signal) = bounded(0);
37 Arc::new(Self {
38 cancelled: AtomicBool::new(false),
39 wait_lock: Mutex::new(()),
40 deadline: Mutex::new(deadline),
41 wake: Condvar::new(),
42 signal,
43 signal_sender: Mutex::new(Some(signal_sender)),
44 #[cfg(test)]
45 require_wait_lock_for_transition: AtomicBool::new(false),
46 })
47 }
48
49 #[cfg(test)]
50 fn assert_transition_holds_wait_lock(&self) {
51 if self.require_wait_lock_for_transition.load(Ordering::SeqCst) {
52 assert!(
53 self.wait_lock.try_lock().is_err(),
54 "predicate transition did not hold the waiter mutex"
55 );
56 }
57 }
58}
59
60#[derive(Clone)]
65pub struct CancellationToken {
66 state: Arc<CancellationState>,
67}
68
69impl fmt::Debug for CancellationToken {
70 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
71 formatter
72 .debug_struct("CancellationToken")
73 .field("cancelled", &self.is_cancelled())
74 .finish_non_exhaustive()
75 }
76}
77
78impl Default for CancellationToken {
79 fn default() -> Self {
80 Self::new()
81 }
82}
83
84impl CancellationToken {
85 pub fn new() -> Self {
87 Self {
88 state: CancellationState::new(None),
89 }
90 }
91
92 pub fn cancel(&self) {
94 let _wait_guard = self.state.wait_lock.lock().unwrap();
95 #[cfg(test)]
96 self.state.assert_transition_holds_wait_lock();
97 if !self.state.cancelled.swap(true, Ordering::AcqRel) {
98 self.state.signal_sender.lock().unwrap().take();
99 self.state.wake.notify_all();
100 }
101 }
102
103 pub fn is_cancelled(&self) -> bool {
105 self.state.cancelled.load(Ordering::Acquire)
106 }
107
108 pub fn wait_cancelled(&self) {
110 let mut guard = self.state.wait_lock.lock().unwrap();
111 while !self.is_cancelled() {
112 guard = self.state.wake.wait(guard).unwrap();
113 }
114 }
115
116 pub fn wait_cancelled_timeout(&self, timeout: Duration) -> bool {
118 if self.is_cancelled() {
119 return true;
120 }
121 let guard = self.state.wait_lock.lock().unwrap();
122 let _ = self
123 .state
124 .wake
125 .wait_timeout_while(guard, timeout, |_| !self.is_cancelled())
126 .unwrap();
127 self.is_cancelled()
128 }
129
130 pub(crate) fn signal(&self) -> &Receiver<()> {
131 &self.state.signal
132 }
133
134 pub(crate) fn paired_deadline(&self) -> Deadline {
135 Deadline {
136 state: Arc::clone(&self.state),
137 }
138 }
139
140 pub(crate) fn wait_cancelled_or_deadline(&self, deadline: &Deadline) -> WaitOutcome {
141 let mut guard = self.state.wait_lock.lock().unwrap();
142 loop {
143 if self.is_cancelled() {
144 return WaitOutcome::Cancelled;
145 }
146 match deadline.remaining() {
147 Some(remaining) if remaining.is_zero() => return WaitOutcome::DeadlineExpired,
148 Some(remaining) => {
149 let (next, _) = self.state.wake.wait_timeout(guard, remaining).unwrap();
150 guard = next;
151 }
152 None => guard = self.state.wake.wait(guard).unwrap(),
153 }
154 }
155 }
156}
157
158#[derive(Clone)]
163pub struct Deadline {
164 state: Arc<CancellationState>,
165}
166
167impl fmt::Debug for Deadline {
168 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
169 formatter
170 .debug_struct("Deadline")
171 .field("remaining", &self.remaining())
172 .finish_non_exhaustive()
173 }
174}
175
176impl Default for Deadline {
177 fn default() -> Self {
178 Self::none()
179 }
180}
181
182impl Deadline {
183 pub fn none() -> Self {
185 Self {
186 state: CancellationState::new(None),
187 }
188 }
189
190 pub fn after(timeout: Duration) -> Self {
195 let now = Instant::now();
196 Self {
197 state: CancellationState::new(Some(now.checked_add(timeout).unwrap_or(now))),
198 }
199 }
200
201 pub fn is_expired(&self) -> bool {
203 self.remaining()
204 .is_some_and(|remaining| remaining.is_zero())
205 }
206
207 pub fn remaining(&self) -> Option<Duration> {
209 self.state
210 .deadline
211 .lock()
212 .unwrap()
213 .map(|deadline| deadline.saturating_duration_since(Instant::now()))
214 }
215
216 pub(crate) fn arm(&self, timeout: Duration) {
217 let _wait_guard = self.state.wait_lock.lock().unwrap();
218 #[cfg(test)]
219 self.state.assert_transition_holds_wait_lock();
220 let now = Instant::now();
221 *self.state.deadline.lock().unwrap() = Some(now.checked_add(timeout).unwrap_or(now));
222 self.state.wake.notify_all();
223 }
224
225 pub(crate) fn disarm(&self) {
226 let _wait_guard = self.state.wait_lock.lock().unwrap();
227 #[cfg(test)]
228 self.state.assert_transition_holds_wait_lock();
229 *self.state.deadline.lock().unwrap() = None;
230 self.state.wake.notify_all();
231 }
232
233 pub(crate) fn can_represent(timeout: Duration) -> bool {
234 Instant::now().checked_add(timeout).is_some()
235 }
236}
237
238#[derive(Clone, Copy, Debug, Eq, PartialEq)]
240pub enum WaitOutcome {
241 Cancelled,
243 DeadlineExpired,
245}
246
247#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)]
249#[error("loader work was cancelled or its deadline expired")]
250pub struct LoaderCancelled;
251
252#[derive(Clone, Copy, Debug, Eq, PartialEq)]
257pub struct WorkerInfo {
258 pub id: usize,
260 pub num_workers: usize,
262 pub seed: u64,
264 pub rank: usize,
266}
267
268impl WorkerInfo {
269 pub fn new(id: usize, num_workers: usize, seed: u64, rank: usize) -> Result<Self> {
276 validate_worker(id, num_workers)?;
277 Ok(Self {
278 id,
279 num_workers,
280 seed,
281 rank,
282 })
283 }
284
285 pub fn from_loader_seed(
296 id: usize,
297 num_workers: usize,
298 loader_seed: u64,
299 rank: usize,
300 iterator_generation: u64,
301 ) -> Result<Self> {
302 Self::new(
303 id,
304 num_workers,
305 derive_worker_seed(loader_seed, rank, iterator_generation, id),
306 rank,
307 )
308 }
309}
310
311#[derive(Clone, Debug)]
349pub struct WorkerContext {
350 pub info: WorkerInfo,
352 pub cancellation: CancellationToken,
354 pub deadline: Deadline,
356}
357
358impl WorkerContext {
359 pub fn new(info: WorkerInfo, cancellation: CancellationToken, deadline: Deadline) -> Self {
361 Self {
362 info,
363 cancellation,
364 deadline,
365 }
366 }
367
368 pub fn check(&self) -> std::result::Result<(), LoaderCancelled> {
370 if self.cancellation.is_cancelled() || self.deadline.is_expired() {
371 Err(LoaderCancelled)
372 } else {
373 Ok(())
374 }
375 }
376
377 pub fn wait_cancelled_or_deadline(&self) -> WaitOutcome {
379 self.cancellation.wait_cancelled_or_deadline(&self.deadline)
380 }
381}
382
383pub trait WorkerInit {
388 type Error;
390
391 fn initialize(&self, worker: &WorkerContext) -> std::result::Result<(), Self::Error>;
393}
394
395#[derive(Clone, Copy, Debug)]
397pub struct FnWorkerInit<F> {
398 initialize: F,
399}
400
401impl<F> FnWorkerInit<F> {
402 pub fn new(initialize: F) -> Self {
404 Self { initialize }
405 }
406}
407
408impl<E, F> WorkerInit for FnWorkerInit<F>
409where
410 F: Fn(&WorkerContext) -> std::result::Result<(), E>,
411{
412 type Error = E;
413
414 fn initialize(&self, worker: &WorkerContext) -> std::result::Result<(), Self::Error> {
415 (self.initialize)(worker)
416 }
417}
418
419#[derive(Clone, Copy, Debug, Default)]
421pub struct NoWorkerInit;
422
423impl WorkerInit for NoWorkerInit {
424 type Error = Infallible;
425
426 fn initialize(&self, _worker: &WorkerContext) -> std::result::Result<(), Self::Error> {
427 Ok(())
428 }
429}
430
431thread_local! {
432 static WORKER_INFO: RefCell<Option<WorkerInfo>> = const { RefCell::new(None) };
433}
434
435pub fn get_worker_info() -> Option<WorkerInfo> {
443 WORKER_INFO.with(|current| *current.borrow())
444}
445
446#[doc(hidden)]
451pub fn with_worker_info<T>(worker: WorkerInfo, callback: impl FnOnce() -> T) -> T {
452 struct Restore(Option<WorkerInfo>);
453
454 impl Drop for Restore {
455 fn drop(&mut self) {
456 WORKER_INFO.with(|current| *current.borrow_mut() = self.0);
457 }
458 }
459
460 let previous = WORKER_INFO.with(|current| current.borrow_mut().replace(worker));
461 let _restore = Restore(previous);
462 callback()
463}
464
465fn validate_worker(id: usize, num_workers: usize) -> Result<()> {
466 if num_workers == 0 {
467 return Err(invalid_configuration(
468 "num_workers",
469 "must be greater than zero",
470 ));
471 }
472 if id >= num_workers {
473 return Err(invalid_configuration(
474 "worker_id",
475 "must be less than num_workers",
476 ));
477 }
478 Ok(())
479}
480
481fn splitmix64(mut value: u64) -> u64 {
482 value = value.wrapping_add(0x9e37_79b9_7f4a_7c15);
483 value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
484 value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
485 value ^ (value >> 31)
486}
487
488fn derive_worker_seed(
489 loader_seed: u64,
490 rank: usize,
491 iterator_generation: u64,
492 worker_id: usize,
493) -> u64 {
494 let rank_domain = (rank as u64).wrapping_mul(0xd1b5_4a32_d192_ed03);
495 let iterator_domain = iterator_generation.wrapping_mul(0xa076_1d64_78bd_642f);
496 splitmix64(loader_seed ^ rank_domain ^ iterator_domain).wrapping_add(worker_id as u64)
497}
498
499fn invalid_configuration(field: &'static str, reason: impl Into<String>) -> RustTorchError {
500 RustTorchError::InvalidConfiguration {
501 field,
502 reason: reason.into(),
503 }
504}
505
506#[cfg(test)]
507mod tests {
508 use super::*;
509
510 #[test]
511 fn cancellation_transition_holds_the_waiter_mutex() {
512 let cancellation = CancellationToken::new();
513 cancellation
514 .state
515 .require_wait_lock_for_transition
516 .store(true, Ordering::SeqCst);
517
518 cancellation.cancel();
519 assert!(cancellation.is_cancelled());
520 }
521
522 #[test]
523 fn deadline_transitions_hold_the_waiter_mutex() {
524 let cancellation = CancellationToken::new();
525 let deadline = cancellation.paired_deadline();
526 cancellation
527 .state
528 .require_wait_lock_for_transition
529 .store(true, Ordering::SeqCst);
530
531 deadline.arm(Duration::from_secs(1));
532 deadline.disarm();
533 assert_eq!(deadline.remaining(), None);
534 }
535}