rusttorch_data/
transform.rs1use std::convert::Infallible;
2
3use rand::SeedableRng;
4use rand_chacha::ChaCha12Rng;
5
6use crate::{
7 CancellationToken, Deadline, LoaderCancelled, Stateless, WaitOutcome, WorkerCheckpoint,
8 WorkerContext,
9};
10
11pub const TASK_RNG_DERIVATION_VERSION: u32 = 1;
13
14#[derive(Clone, Debug)]
20pub struct TaskContext {
21 pub loader_seed: u64,
23 pub epoch: u64,
25 pub rank: usize,
27 pub logical_sample: u64,
29 pub stage: u32,
31 pub cancellation: CancellationToken,
33 pub deadline: Deadline,
35}
36
37impl TaskContext {
38 pub fn deterministic_seed(&self) -> u64 {
40 derive_task_seed(self)
41 }
42
43 pub fn rng(&self) -> ChaCha12Rng {
45 ChaCha12Rng::seed_from_u64(self.deterministic_seed())
46 }
47
48 pub fn check(&self) -> std::result::Result<(), LoaderCancelled> {
50 if self.cancellation.is_cancelled() || self.deadline.is_expired() {
51 Err(LoaderCancelled)
52 } else {
53 Ok(())
54 }
55 }
56
57 pub fn wait_cancelled_or_deadline(&self) -> WaitOutcome {
59 self.cancellation.wait_cancelled_or_deadline(&self.deadline)
60 }
61}
62
63pub trait Transform<Input> {
65 type Output;
67 type Error;
69
70 fn transform(
72 &mut self,
73 input: Input,
74 context: &TaskContext,
75 ) -> std::result::Result<Self::Output, Self::Error>;
76}
77
78#[derive(Clone, Copy, Debug)]
100pub struct FnTransform<F> {
101 transform: F,
102}
103
104impl<F> FnTransform<F> {
105 pub fn new(transform: F) -> Self {
107 Self { transform }
108 }
109}
110
111impl<Input, Output, E, F> Transform<Input> for FnTransform<F>
112where
113 F: FnMut(Input, &TaskContext) -> std::result::Result<Output, E>,
114{
115 type Output = Output;
116 type Error = E;
117
118 fn transform(
119 &mut self,
120 input: Input,
121 context: &TaskContext,
122 ) -> std::result::Result<Self::Output, Self::Error> {
123 (self.transform)(input, context)
124 }
125}
126
127pub trait TransformFactory<Input> {
132 type Transform: Transform<Input>;
134 type Error;
136
137 fn create(
142 &self,
143 worker: Option<&WorkerContext>,
144 ) -> std::result::Result<Self::Transform, Self::Error>;
145}
146
147#[derive(Clone, Copy, Debug)]
149pub struct FnTransformFactory<F> {
150 create: F,
151}
152
153impl<F> FnTransformFactory<F> {
154 pub fn new(create: F) -> Self {
156 Self { create }
157 }
158}
159
160impl<Input, T, E, F> TransformFactory<Input> for FnTransformFactory<F>
161where
162 T: Transform<Input>,
163 F: Fn(Option<&WorkerContext>) -> std::result::Result<T, E>,
164{
165 type Transform = T;
166 type Error = E;
167
168 fn create(
169 &self,
170 worker: Option<&WorkerContext>,
171 ) -> std::result::Result<Self::Transform, Self::Error> {
172 (self.create)(worker)
173 }
174}
175
176#[derive(Clone, Copy, Debug)]
178pub struct CloneTransformFactory<T> {
179 transform: T,
180}
181
182impl<T> CloneTransformFactory<T> {
183 pub(crate) fn new(transform: T) -> Self {
184 Self { transform }
185 }
186}
187
188impl<Input, T> TransformFactory<Input> for CloneTransformFactory<T>
189where
190 T: Transform<Input> + Clone,
191{
192 type Transform = T;
193 type Error = Infallible;
194
195 fn create(
196 &self,
197 _worker: Option<&WorkerContext>,
198 ) -> std::result::Result<Self::Transform, Self::Error> {
199 Ok(self.transform.clone())
200 }
201}
202
203#[derive(Clone, Copy, Debug, Default)]
205pub struct IdentityTransform;
206
207impl Stateless for IdentityTransform {}
208
209impl WorkerCheckpoint for IdentityTransform {
210 type State = ();
211
212 fn snapshot(&self) -> Self::State {}
213
214 fn validate_snapshot(&self, _state: &Self::State) -> rusttorch_core::Result<()> {
215 Ok(())
216 }
217
218 fn restore_validated(&mut self, _state: &Self::State) {}
219}
220
221impl<Input> Transform<Input> for IdentityTransform {
222 type Output = Input;
223 type Error = Infallible;
224
225 fn transform(
226 &mut self,
227 input: Input,
228 _context: &TaskContext,
229 ) -> std::result::Result<Self::Output, Self::Error> {
230 Ok(input)
231 }
232}
233
234#[derive(Clone, Copy, Debug, Default)]
236pub struct IdentityTransformFactory;
237
238impl<Input> TransformFactory<Input> for IdentityTransformFactory {
239 type Transform = IdentityTransform;
240 type Error = Infallible;
241
242 fn create(
243 &self,
244 _worker: Option<&WorkerContext>,
245 ) -> std::result::Result<Self::Transform, Self::Error> {
246 Ok(IdentityTransform)
247 }
248}
249
250fn splitmix64(mut value: u64) -> u64 {
251 value = value.wrapping_add(0x9e37_79b9_7f4a_7c15);
252 value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
253 value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
254 value ^ (value >> 31)
255}
256
257fn derive_task_seed(context: &TaskContext) -> u64 {
258 let fields = [
259 context.loader_seed,
260 context.epoch,
261 context.rank as u64,
262 context.logical_sample,
263 u64::from(context.stage),
264 ];
265 fields
266 .into_iter()
267 .enumerate()
268 .fold(0x5255_5354_544f_5243, |state, (index, value)| {
269 let domain = (index as u64 + 1).wrapping_mul(0x9e37_79b9_7f4a_7c15);
270 splitmix64(state ^ value.wrapping_add(domain))
271 })
272}