use std::{
cell::RefCell,
convert::Infallible,
fmt,
sync::{
Arc, Condvar, Mutex,
atomic::{AtomicBool, Ordering},
},
time::{Duration, Instant},
};
use crossbeam_channel::{Receiver, Sender, bounded};
use rusttorch_core::{Result, RustTorchError};
pub const WORKER_SEED_DERIVATION_VERSION: u32 = 1;
struct CancellationState {
cancelled: AtomicBool,
wait_lock: Mutex<()>,
deadline: Mutex<Option<Instant>>,
wake: Condvar,
signal: Receiver<()>,
signal_sender: Mutex<Option<Sender<()>>>,
#[cfg(test)]
require_wait_lock_for_transition: AtomicBool,
}
impl CancellationState {
fn new(deadline: Option<Instant>) -> Arc<Self> {
let (signal_sender, signal) = bounded(0);
Arc::new(Self {
cancelled: AtomicBool::new(false),
wait_lock: Mutex::new(()),
deadline: Mutex::new(deadline),
wake: Condvar::new(),
signal,
signal_sender: Mutex::new(Some(signal_sender)),
#[cfg(test)]
require_wait_lock_for_transition: AtomicBool::new(false),
})
}
#[cfg(test)]
fn assert_transition_holds_wait_lock(&self) {
if self.require_wait_lock_for_transition.load(Ordering::SeqCst) {
assert!(
self.wait_lock.try_lock().is_err(),
"predicate transition did not hold the waiter mutex"
);
}
}
}
#[derive(Clone)]
pub struct CancellationToken {
state: Arc<CancellationState>,
}
impl fmt::Debug for CancellationToken {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("CancellationToken")
.field("cancelled", &self.is_cancelled())
.finish_non_exhaustive()
}
}
impl Default for CancellationToken {
fn default() -> Self {
Self::new()
}
}
impl CancellationToken {
pub fn new() -> Self {
Self {
state: CancellationState::new(None),
}
}
pub fn cancel(&self) {
let _wait_guard = self.state.wait_lock.lock().unwrap();
#[cfg(test)]
self.state.assert_transition_holds_wait_lock();
if !self.state.cancelled.swap(true, Ordering::AcqRel) {
self.state.signal_sender.lock().unwrap().take();
self.state.wake.notify_all();
}
}
pub fn is_cancelled(&self) -> bool {
self.state.cancelled.load(Ordering::Acquire)
}
pub fn wait_cancelled(&self) {
let mut guard = self.state.wait_lock.lock().unwrap();
while !self.is_cancelled() {
guard = self.state.wake.wait(guard).unwrap();
}
}
pub fn wait_cancelled_timeout(&self, timeout: Duration) -> bool {
if self.is_cancelled() {
return true;
}
let guard = self.state.wait_lock.lock().unwrap();
let _ = self
.state
.wake
.wait_timeout_while(guard, timeout, |_| !self.is_cancelled())
.unwrap();
self.is_cancelled()
}
pub(crate) fn signal(&self) -> &Receiver<()> {
&self.state.signal
}
pub(crate) fn paired_deadline(&self) -> Deadline {
Deadline {
state: Arc::clone(&self.state),
}
}
pub(crate) fn wait_cancelled_or_deadline(&self, deadline: &Deadline) -> WaitOutcome {
let mut guard = self.state.wait_lock.lock().unwrap();
loop {
if self.is_cancelled() {
return WaitOutcome::Cancelled;
}
match deadline.remaining() {
Some(remaining) if remaining.is_zero() => return WaitOutcome::DeadlineExpired,
Some(remaining) => {
let (next, _) = self.state.wake.wait_timeout(guard, remaining).unwrap();
guard = next;
}
None => guard = self.state.wake.wait(guard).unwrap(),
}
}
}
}
#[derive(Clone)]
pub struct Deadline {
state: Arc<CancellationState>,
}
impl fmt::Debug for Deadline {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("Deadline")
.field("remaining", &self.remaining())
.finish_non_exhaustive()
}
}
impl Default for Deadline {
fn default() -> Self {
Self::none()
}
}
impl Deadline {
pub fn none() -> Self {
Self {
state: CancellationState::new(None),
}
}
pub fn after(timeout: Duration) -> Self {
let now = Instant::now();
Self {
state: CancellationState::new(Some(now.checked_add(timeout).unwrap_or(now))),
}
}
pub fn is_expired(&self) -> bool {
self.remaining()
.is_some_and(|remaining| remaining.is_zero())
}
pub fn remaining(&self) -> Option<Duration> {
self.state
.deadline
.lock()
.unwrap()
.map(|deadline| deadline.saturating_duration_since(Instant::now()))
}
pub(crate) fn arm(&self, timeout: Duration) {
let _wait_guard = self.state.wait_lock.lock().unwrap();
#[cfg(test)]
self.state.assert_transition_holds_wait_lock();
let now = Instant::now();
*self.state.deadline.lock().unwrap() = Some(now.checked_add(timeout).unwrap_or(now));
self.state.wake.notify_all();
}
pub(crate) fn disarm(&self) {
let _wait_guard = self.state.wait_lock.lock().unwrap();
#[cfg(test)]
self.state.assert_transition_holds_wait_lock();
*self.state.deadline.lock().unwrap() = None;
self.state.wake.notify_all();
}
pub(crate) fn can_represent(timeout: Duration) -> bool {
Instant::now().checked_add(timeout).is_some()
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum WaitOutcome {
Cancelled,
DeadlineExpired,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)]
#[error("loader work was cancelled or its deadline expired")]
pub struct LoaderCancelled;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct WorkerInfo {
pub id: usize,
pub num_workers: usize,
pub seed: u64,
pub rank: usize,
}
impl WorkerInfo {
pub fn new(id: usize, num_workers: usize, seed: u64, rank: usize) -> Result<Self> {
validate_worker(id, num_workers)?;
Ok(Self {
id,
num_workers,
seed,
rank,
})
}
pub fn from_loader_seed(
id: usize,
num_workers: usize,
loader_seed: u64,
rank: usize,
iterator_generation: u64,
) -> Result<Self> {
Self::new(
id,
num_workers,
derive_worker_seed(loader_seed, rank, iterator_generation, id),
rank,
)
}
}
#[derive(Clone, Debug)]
pub struct WorkerContext {
pub info: WorkerInfo,
pub cancellation: CancellationToken,
pub deadline: Deadline,
}
impl WorkerContext {
pub fn new(info: WorkerInfo, cancellation: CancellationToken, deadline: Deadline) -> Self {
Self {
info,
cancellation,
deadline,
}
}
pub fn check(&self) -> std::result::Result<(), LoaderCancelled> {
if self.cancellation.is_cancelled() || self.deadline.is_expired() {
Err(LoaderCancelled)
} else {
Ok(())
}
}
pub fn wait_cancelled_or_deadline(&self) -> WaitOutcome {
self.cancellation.wait_cancelled_or_deadline(&self.deadline)
}
}
pub trait WorkerInit {
type Error;
fn initialize(&self, worker: &WorkerContext) -> std::result::Result<(), Self::Error>;
}
#[derive(Clone, Copy, Debug)]
pub struct FnWorkerInit<F> {
initialize: F,
}
impl<F> FnWorkerInit<F> {
pub fn new(initialize: F) -> Self {
Self { initialize }
}
}
impl<E, F> WorkerInit for FnWorkerInit<F>
where
F: Fn(&WorkerContext) -> std::result::Result<(), E>,
{
type Error = E;
fn initialize(&self, worker: &WorkerContext) -> std::result::Result<(), Self::Error> {
(self.initialize)(worker)
}
}
#[derive(Clone, Copy, Debug, Default)]
pub struct NoWorkerInit;
impl WorkerInit for NoWorkerInit {
type Error = Infallible;
fn initialize(&self, _worker: &WorkerContext) -> std::result::Result<(), Self::Error> {
Ok(())
}
}
thread_local! {
static WORKER_INFO: RefCell<Option<WorkerInfo>> = const { RefCell::new(None) };
}
pub fn get_worker_info() -> Option<WorkerInfo> {
WORKER_INFO.with(|current| *current.borrow())
}
#[doc(hidden)]
pub fn with_worker_info<T>(worker: WorkerInfo, callback: impl FnOnce() -> T) -> T {
struct Restore(Option<WorkerInfo>);
impl Drop for Restore {
fn drop(&mut self) {
WORKER_INFO.with(|current| *current.borrow_mut() = self.0);
}
}
let previous = WORKER_INFO.with(|current| current.borrow_mut().replace(worker));
let _restore = Restore(previous);
callback()
}
fn validate_worker(id: usize, num_workers: usize) -> Result<()> {
if num_workers == 0 {
return Err(invalid_configuration(
"num_workers",
"must be greater than zero",
));
}
if id >= num_workers {
return Err(invalid_configuration(
"worker_id",
"must be less than num_workers",
));
}
Ok(())
}
fn splitmix64(mut value: u64) -> u64 {
value = value.wrapping_add(0x9e37_79b9_7f4a_7c15);
value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
value ^ (value >> 31)
}
fn derive_worker_seed(
loader_seed: u64,
rank: usize,
iterator_generation: u64,
worker_id: usize,
) -> u64 {
let rank_domain = (rank as u64).wrapping_mul(0xd1b5_4a32_d192_ed03);
let iterator_domain = iterator_generation.wrapping_mul(0xa076_1d64_78bd_642f);
splitmix64(loader_seed ^ rank_domain ^ iterator_domain).wrapping_add(worker_id as u64)
}
fn invalid_configuration(field: &'static str, reason: impl Into<String>) -> RustTorchError {
RustTorchError::InvalidConfiguration {
field,
reason: reason.into(),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn cancellation_transition_holds_the_waiter_mutex() {
let cancellation = CancellationToken::new();
cancellation
.state
.require_wait_lock_for_transition
.store(true, Ordering::SeqCst);
cancellation.cancel();
assert!(cancellation.is_cancelled());
}
#[test]
fn deadline_transitions_hold_the_waiter_mutex() {
let cancellation = CancellationToken::new();
let deadline = cancellation.paired_deadline();
cancellation
.state
.require_wait_lock_for_transition
.store(true, Ordering::SeqCst);
deadline.arm(Duration::from_secs(1));
deadline.disarm();
assert_eq!(deadline.remaining(), None);
}
}