Skip to main content

rusttorch_data/
transform.rs

1use std::convert::Infallible;
2
3use rand::SeedableRng;
4use rand_chacha::ChaCha12Rng;
5
6use crate::{
7    CancellationToken, Deadline, LoaderCancelled, Stateless, WaitOutcome, WorkerCheckpoint,
8    WorkerContext,
9};
10
11/// Version of RustTorch's deterministic task-RNG seed derivation.
12pub const TASK_RNG_DERIVATION_VERSION: u32 = 1;
13
14/// Logical identity used to seed one transformation occurrence.
15///
16/// Worker assignment is deliberately absent, so random transforms are stable
17/// when scheduling or worker counts change. The resulting ChaCha12 sequence is
18/// a RustTorch contract and does not claim PyTorch Philox sequence identity.
19#[derive(Clone, Debug)]
20pub struct TaskContext {
21    /// Loader-level seed.
22    pub loader_seed: u64,
23    /// Current data epoch.
24    pub epoch: u64,
25    /// Distributed rank.
26    pub rank: usize,
27    /// Monotonic logical sample occurrence in this iterator.
28    pub logical_sample: u64,
29    /// Transform stage, starting at zero.
30    pub stage: u32,
31    /// Cancellation for the active iterator generation.
32    pub cancellation: CancellationToken,
33    /// Dynamic deadline for the active `Iterator::next` wait.
34    pub deadline: Deadline,
35}
36
37impl TaskContext {
38    /// Returns the stable version-1 seed derived from this logical identity.
39    pub fn deterministic_seed(&self) -> u64 {
40        derive_task_seed(self)
41    }
42
43    /// Creates task-local ChaCha12 state without changing LibTorch's RNG.
44    pub fn rng(&self) -> ChaCha12Rng {
45        ChaCha12Rng::seed_from_u64(self.deterministic_seed())
46    }
47
48    /// Returns an error when this generation is cancelled or expired.
49    pub fn check(&self) -> std::result::Result<(), LoaderCancelled> {
50        if self.cancellation.is_cancelled() || self.deadline.is_expired() {
51            Err(LoaderCancelled)
52        } else {
53            Ok(())
54        }
55    }
56
57    /// Blocks until generation cancellation or deadline expiry.
58    pub fn wait_cancelled_or_deadline(&self) -> WaitOutcome {
59        self.cancellation.wait_cancelled_or_deadline(&self.deadline)
60    }
61}
62
63/// A fallible sample transformation with task-local deterministic context.
64pub trait Transform<Input> {
65    /// Output passed to collation.
66    type Output;
67    /// Transformation failure.
68    type Error;
69
70    /// Transforms one logical sample occurrence.
71    fn transform(
72        &mut self,
73        input: Input,
74        context: &TaskContext,
75    ) -> std::result::Result<Self::Output, Self::Error>;
76}
77
78/// Adapts a fallible closure into a [`Transform`].
79///
80/// ```
81/// use std::convert::Infallible;
82/// use rusttorch_data::{FnTransform, TaskContext, Transform};
83///
84/// let context = TaskContext {
85///     loader_seed: 42,
86///     epoch: 3,
87///     rank: 1,
88///     logical_sample: 99,
89///     stage: 7,
90///     cancellation: rusttorch_data::CancellationToken::new(),
91///     deadline: rusttorch_data::Deadline::none(),
92/// };
93/// let mut double = FnTransform::new(|value: i64, _: &TaskContext| {
94///     Ok::<_, Infallible>(value * 2)
95/// });
96/// assert_eq!(double.transform(3, &context), Ok(6));
97/// assert_eq!(context.deterministic_seed(), 0x1d7d_73dc_f6e9_4f2d);
98/// ```
99#[derive(Clone, Copy, Debug)]
100pub struct FnTransform<F> {
101    transform: F,
102}
103
104impl<F> FnTransform<F> {
105    /// Creates a closure-backed transform.
106    pub fn new(transform: F) -> Self {
107        Self { transform }
108    }
109}
110
111impl<Input, Output, E, F> Transform<Input> for FnTransform<F>
112where
113    F: FnMut(Input, &TaskContext) -> std::result::Result<Output, E>,
114{
115    type Output = Output;
116    type Error = E;
117
118    fn transform(
119        &mut self,
120        input: Input,
121        context: &TaskContext,
122    ) -> std::result::Result<Self::Output, Self::Error> {
123        (self.transform)(input, context)
124    }
125}
126
127/// Constructs one transform for a serial iterator or worker.
128///
129/// Local factories and transforms may remain non-`Send` on the default serial
130/// loader. Selecting positive workers adds the thread-safety bounds.
131pub trait TransformFactory<Input> {
132    /// Transform instance created by this factory.
133    type Transform: Transform<Input>;
134    /// Construction failure.
135    type Error;
136
137    /// Creates a transform for `worker`, or for the serial path when `None`.
138    ///
139    /// A persistent worker receives its pool-lifetime context here. Individual
140    /// task transforms receive generation cancellation through [`TaskContext`].
141    fn create(
142        &self,
143        worker: Option<&WorkerContext>,
144    ) -> std::result::Result<Self::Transform, Self::Error>;
145}
146
147/// Adapts a fallible closure into a [`TransformFactory`].
148#[derive(Clone, Copy, Debug)]
149pub struct FnTransformFactory<F> {
150    create: F,
151}
152
153impl<F> FnTransformFactory<F> {
154    /// Creates a closure-backed factory.
155    pub fn new(create: F) -> Self {
156        Self { create }
157    }
158}
159
160impl<Input, T, E, F> TransformFactory<Input> for FnTransformFactory<F>
161where
162    T: Transform<Input>,
163    F: Fn(Option<&WorkerContext>) -> std::result::Result<T, E>,
164{
165    type Transform = T;
166    type Error = E;
167
168    fn create(
169        &self,
170        worker: Option<&WorkerContext>,
171    ) -> std::result::Result<Self::Transform, Self::Error> {
172        (self.create)(worker)
173    }
174}
175
176/// Factory used by [`crate::DataLoaderBuilder::transform`] to clone a template.
177#[derive(Clone, Copy, Debug)]
178pub struct CloneTransformFactory<T> {
179    transform: T,
180}
181
182impl<T> CloneTransformFactory<T> {
183    pub(crate) fn new(transform: T) -> Self {
184        Self { transform }
185    }
186}
187
188impl<Input, T> TransformFactory<Input> for CloneTransformFactory<T>
189where
190    T: Transform<Input> + Clone,
191{
192    type Transform = T;
193    type Error = Infallible;
194
195    fn create(
196        &self,
197        _worker: Option<&WorkerContext>,
198    ) -> std::result::Result<Self::Transform, Self::Error> {
199        Ok(self.transform.clone())
200    }
201}
202
203/// Identity sample transform used by the default factory.
204#[derive(Clone, Copy, Debug, Default)]
205pub struct IdentityTransform;
206
207impl Stateless for IdentityTransform {}
208
209impl WorkerCheckpoint for IdentityTransform {
210    type State = ();
211
212    fn snapshot(&self) -> Self::State {}
213
214    fn validate_snapshot(&self, _state: &Self::State) -> rusttorch_core::Result<()> {
215        Ok(())
216    }
217
218    fn restore_validated(&mut self, _state: &Self::State) {}
219}
220
221impl<Input> Transform<Input> for IdentityTransform {
222    type Output = Input;
223    type Error = Infallible;
224
225    fn transform(
226        &mut self,
227        input: Input,
228        _context: &TaskContext,
229    ) -> std::result::Result<Self::Output, Self::Error> {
230        Ok(input)
231    }
232}
233
234/// Default factory that creates infallible identity transforms.
235#[derive(Clone, Copy, Debug, Default)]
236pub struct IdentityTransformFactory;
237
238impl<Input> TransformFactory<Input> for IdentityTransformFactory {
239    type Transform = IdentityTransform;
240    type Error = Infallible;
241
242    fn create(
243        &self,
244        _worker: Option<&WorkerContext>,
245    ) -> std::result::Result<Self::Transform, Self::Error> {
246        Ok(IdentityTransform)
247    }
248}
249
250fn splitmix64(mut value: u64) -> u64 {
251    value = value.wrapping_add(0x9e37_79b9_7f4a_7c15);
252    value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
253    value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
254    value ^ (value >> 31)
255}
256
257fn derive_task_seed(context: &TaskContext) -> u64 {
258    let fields = [
259        context.loader_seed,
260        context.epoch,
261        context.rank as u64,
262        context.logical_sample,
263        u64::from(context.stage),
264    ];
265    fields
266        .into_iter()
267        .enumerate()
268        .fold(0x5255_5354_544f_5243, |state, (index, value)| {
269            let domain = (index as u64 + 1).wrapping_mul(0x9e37_79b9_7f4a_7c15);
270            splitmix64(state ^ value.wrapping_add(domain))
271        })
272}