Skip to main content

rusttorch_data/
checkpoint.rs

1use std::{error::Error, fmt};
2
3use rusttorch_core::{Result, RustTorchError};
4use serde::{Deserialize, Serialize, de::DeserializeOwned};
5
6use crate::{TaskContext, Transform, TransformFactory, WorkerContext};
7
8/// Replay-safe source with self-contained state captured before each read.
9///
10/// Validation is read-only; applying accepted state must not fail or panic.
11/// Records must have strictly increasing per-shard global sequence IDs.
12pub trait CheckpointableSource:
13    Iterator<Item = std::result::Result<crate::WorkerRecord<Self::Sample>, Self::Error>>
14{
15    /// Owned decoded sample.
16    type Sample;
17    /// Owned state, independent of future decoder progress.
18    type State: Clone + Serialize + DeserializeOwned;
19    /// Preserved source error.
20    type Error;
21    /// Returns the failed read's stable global position without changing state.
22    ///
23    /// Called after `next()` returns an error. This position must obey the
24    /// shard's strictly increasing sequence contract and repeat after rollback.
25    fn error_sequence(&self, error: &Self::Error) -> crate::SequenceId;
26    /// Captures the boundary before the next read attempt.
27    fn snapshot(&self) -> Self::State;
28    /// Validates without changing the source.
29    fn validate_snapshot(&self, state: &Self::State) -> std::result::Result<(), Self::Error>;
30    /// Applies state previously accepted by validation.
31    fn restore_validated(&mut self, state: &Self::State);
32    /// Replaces the run cancellation/deadline context before resumed reads.
33    ///
34    /// Sources retaining a context must override this method and replace it.
35    /// Logical worker generation and seed remain unchanged. This must not fail.
36    fn set_run_context(&mut self, _context: WorkerContext) {}
37}
38
39/// Stable factory identity for explicitly checkpointable stream sources.
40///
41/// An ordinary factory cannot acquire exact replay just by setting an identity:
42///
43/// ```compile_fail
44/// use rusttorch_data::*;
45/// use std::convert::Infallible;
46/// struct Factory;
47/// impl WorkerSourceFactory for Factory {
48///     type Sample = usize;
49///     type Error = Infallible;
50///     type Source = std::iter::Empty<Result<WorkerRecord<usize>, Infallible>>;
51///     fn create(&self, _: WorkerContext) -> Result<Self::Source, Infallible> { Ok(std::iter::empty()) }
52/// }
53/// StreamDataLoaderBuilder::new(Factory).collate(VecCollate)
54///     .checkpointable("ordinary").build().unwrap();
55/// ```
56///
57/// Opaque transforms also need an explicit `WorkerCheckpoint` implementation or
58/// an explicit `StatelessWorker`/`TransactionalWorker` adapter:
59///
60/// ```compile_fail
61/// use rusttorch_data::*;
62/// use std::convert::Infallible;
63/// struct Source;
64/// impl Iterator for Source {
65///     type Item = Result<WorkerRecord<usize>, Infallible>;
66///     fn next(&mut self) -> Option<Self::Item> { None }
67/// }
68/// impl CheckpointableSource for Source {
69///     type Sample = usize;
70///     type Error = Infallible;
71///     type State = ();
72///     fn error_sequence(&self, error: &Infallible) -> SequenceId { match *error {} }
73///     fn snapshot(&self) {}
74///     fn validate_snapshot(&self, _: &()) -> Result<(), Infallible> { Ok(()) }
75///     fn restore_validated(&mut self, _: &()) {}
76/// }
77/// struct Factory;
78/// impl WorkerSourceFactory for Factory {
79///     type Sample = usize;
80///     type Error = Infallible;
81///     type Source = Source;
82///     fn create(&self, _: WorkerContext) -> Result<Source, Infallible> { Ok(Source) }
83/// }
84/// impl CheckpointSourceFactory for Factory { const CHECKPOINT_KIND: &'static str = "empty.v1"; }
85/// let transform = FnTransform::new(|sample: usize, _: &TaskContext| Ok::<_, Infallible>(sample));
86/// StreamDataLoaderBuilder::new(Factory).transform(transform).collate(VecCollate)
87///     .checkpointable("opaque").build().unwrap();
88/// ```
89pub trait CheckpointSourceFactory: crate::WorkerSourceFactory {
90    /// Stable source kind and state-format version, for example `records.v1`.
91    const CHECKPOINT_KIND: &'static str;
92}
93
94/// Exact streaming configuration, with no sampler state.
95#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
96pub struct StreamCheckpointConfiguration {
97    /// Positive shard count.
98    pub workers: usize,
99    /// Maximum unpublished records per shard.
100    pub prefetch_factor: usize,
101    /// Coordinator batch size.
102    pub batch_size: usize,
103    /// Global short-tail policy.
104    pub drop_last: bool,
105    /// Task randomness seed.
106    pub loader_seed: u64,
107    /// Distributed task rank.
108    pub rank: usize,
109    /// Requested pinning behavior.
110    pub pin_request: CheckpointPinRequest,
111    /// Effective pinning behavior.
112    pub pin_status: CheckpointPinStatus,
113}
114
115/// Paired source and transform boundary for one worker shard.
116#[derive(Clone, Debug, Deserialize, Serialize)]
117pub struct StreamLaneState<S, T> {
118    /// Unique worker index in canonical order.
119    pub id: usize,
120    /// Source state preceding its first unconsumed attempt.
121    pub source: S,
122    /// Transform state preceding the same attempt.
123    pub transform: T,
124}
125
126/// Versioned exact stream state at the next consumer-visible batch.
127#[derive(Clone, Debug, Deserialize, Serialize)]
128pub struct StreamLoaderState<S, T = (), C = ()> {
129    /// Stream schema version; currently one.
130    pub schema_version: u32,
131    /// Caller identity for exact source contents.
132    pub source_identity: String,
133    /// Stable factory kind/version.
134    pub factory_kind: String,
135    /// Task epoch.
136    pub epoch: u64,
137    /// Logical source and transform initialization generation.
138    pub source_generation: u64,
139    /// Transport generation, advanced by each checkpoint barrier.
140    pub transport_generation: u64,
141    /// Next visible batch index.
142    pub next_batch: u64,
143    /// Next expected global sequence ID.
144    pub next_sequence: u64,
145    /// Whether the last visible batch was the final incomplete global tail.
146    pub short_tail: bool,
147    /// Exact global source length when supplied by the factory.
148    pub source_length: Option<usize>,
149    /// Task RNG derivation version.
150    pub rng_derivation_version: u32,
151    /// Worker seed derivation version.
152    pub worker_seed_derivation_version: u32,
153    /// Exact runtime settings.
154    pub configuration: StreamCheckpointConfiguration,
155    /// Canonically ordered worker states; never contains decoded samples.
156    pub lanes: Vec<StreamLaneState<S, T>>,
157    /// Coordinator collation state.
158    pub coordinator: C,
159}
160
161/// Typed stream construction or transactional resume failure.
162#[derive(Debug, thiserror::Error)]
163pub enum StreamCheckpointBuildError<S, F> {
164    /// Static configuration rejection.
165    #[error(transparent)]
166    Configuration(#[from] RustTorchError),
167    /// Source creation or read-only state validation failed.
168    #[error("stream source on worker {worker} failed: {source}")]
169    Source {
170        /// Worker lane.
171        worker: usize,
172        /// Original source error.
173        #[source]
174        source: S,
175    },
176    /// Transform creation failed.
177    #[error("stream transform initialization on worker {worker} failed: {source}")]
178    TransformFactory {
179        /// Worker lane.
180        worker: usize,
181        /// Original factory error.
182        #[source]
183        source: F,
184    },
185    /// Transform state validation failed.
186    #[error("stream transform validation on worker {worker} failed: {source}")]
187    TransformState {
188        /// Worker lane.
189        worker: usize,
190        /// Original validation error.
191        #[source]
192        source: RustTorchError,
193    },
194    /// Initialization or checkpoint callback panicked.
195    #[error("stream checkpoint worker {worker} panicked")]
196    WorkerPanic {
197        /// Worker lane.
198        worker: usize,
199    },
200}
201
202/// Schema version emitted by serial map-loader checkpoints.
203pub const LOADER_STATE_SCHEMA_VERSION: u32 = 1;
204
205/// Pinning request recorded in a loader checkpoint.
206#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
207pub enum CheckpointPinRequest {
208    /// Pinning was not requested.
209    Disabled,
210    /// Pinning was requested only when a CUDA accelerator is available.
211    Auto,
212    /// Pinning was requested for one CUDA device.
213    ExplicitCuda {
214        /// CUDA device index.
215        index: usize,
216    },
217}
218
219/// Effective pinning behavior recorded in a loader checkpoint.
220#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
221pub enum CheckpointPinStatus {
222    /// Pinning was not requested.
223    Disabled,
224    /// Automatic pinning found no supported accelerator.
225    DisabledNoAccelerator,
226    /// Batches are pinned for one CUDA device.
227    Cuda {
228        /// CUDA device index.
229        index: usize,
230    },
231}
232
233/// Loader settings that must match when a serial checkpoint is resumed.
234#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
235pub struct LoaderConfiguration {
236    /// Automatic batch size, or `None` for explicit/no batching.
237    pub batch_size: Option<usize>,
238    /// Whether an incomplete automatic batch is omitted.
239    pub drop_last: bool,
240    /// Exact worker count; zero for serial execution.
241    pub workers: usize,
242    /// Effective worker prefetch factor, or `None` for serial execution.
243    pub prefetch_factor: Option<usize>,
244    /// Whether results follow sampler order.
245    pub in_order: bool,
246    /// Loader seed used by task-local random transforms.
247    pub loader_seed: u64,
248    /// Distributed task rank.
249    pub rank: usize,
250    /// Distributed replica count, or one for an unsharded sampler.
251    pub replicas: usize,
252    /// Distributed shuffle policy when applicable.
253    pub distributed_shuffle: Option<bool>,
254    /// Distributed sampler seed when applicable.
255    pub distributed_seed: Option<u64>,
256    /// Distributed tail policy when applicable.
257    pub distributed_drop_last: Option<bool>,
258    /// Stable RustTorch sampler/plan kind.
259    pub sampler_kind: String,
260    /// Requested pinning policy.
261    pub pin_request: CheckpointPinRequest,
262    /// Effective pinning status visible to consumers.
263    pub pin_status: CheckpointPinStatus,
264}
265
266/// Versioned state for the next batch visible from a map loader.
267#[derive(Clone, Debug, Deserialize, Serialize)]
268pub struct LoaderState<D, S, T = (), C = ()> {
269    /// State schema version.
270    pub schema_version: u32,
271    /// Caller-defined identity for the exact dataset contents.
272    pub dataset_identity: String,
273    /// Active sampler epoch.
274    pub epoch: u64,
275    /// Transport generation. Serial state is always zero.
276    pub iterator_generation: u64,
277    /// Next consumer-visible batch number.
278    pub next_batch: u64,
279    /// Number of logical sample occurrences already consumed.
280    pub next_logical_sample: u64,
281    /// Dataset component state.
282    pub dataset: D,
283    /// Active sampler or batch-source cursor state.
284    pub sampler: S,
285    /// Serial transform state or a versioned worker transform envelope.
286    pub transform: T,
287    /// Effective coordinator collator/converter state.
288    pub collate: C,
289    /// Task-RNG derivation version.
290    pub rng_derivation_version: u32,
291    /// Worker-seed derivation version.
292    pub worker_seed_derivation_version: u32,
293    /// Exact loader configuration identity.
294    pub configuration: LoaderConfiguration,
295}
296
297/// A component whose state can be validated before it is restored.
298///
299/// Validation must be read-only. `load_validated` must not fail or panic after
300/// the same component accepted the supplied state; violating either rule
301/// breaks the resume transaction contract.
302pub trait Checkpointable {
303    /// Owned serializable component state.
304    type State: Clone + Serialize + DeserializeOwned;
305
306    /// Captures the current component state.
307    fn save_state(&self) -> Self::State;
308
309    /// Checks state compatibility without mutating the component.
310    fn validate_state(&self, state: &Self::State) -> Result<()>;
311
312    /// Restores state previously accepted by [`Self::validate_state`].
313    fn load_validated(&mut self, state: &Self::State);
314}
315
316/// Marker for a map dataset whose fixed-identity fetches are replay-safe.
317pub trait ReplaySafeDataset: crate::Dataset {}
318
319/// Dataset state used by exact serial loader checkpoints.
320///
321/// Validation must be read-only, and restore must be infallible after matching
322/// validation succeeds.
323pub trait DatasetCheckpoint: crate::Dataset {
324    /// Owned serializable dataset state.
325    type State: Clone + Serialize + DeserializeOwned;
326
327    /// Captures dataset state at the visible boundary.
328    fn snapshot_dataset(&self) -> Self::State;
329
330    /// Checks state compatibility without mutation.
331    fn validate_dataset_state(&self, state: &Self::State) -> Result<()>;
332
333    /// Restores state accepted by [`Self::validate_dataset_state`].
334    fn restore_dataset_validated(&mut self, state: &Self::State);
335}
336
337/// Marker for a component with no mutable checkpoint state.
338pub trait Stateless {}
339
340/// Transactional state contract for a serial component.
341///
342/// Validation must be read-only, and restore must be infallible after matching
343/// validation succeeds.
344pub trait TransactionalCheckpoint {
345    /// Owned serializable component state.
346    type State: Clone + Serialize + DeserializeOwned;
347
348    /// Captures current component state.
349    fn snapshot(&self) -> Self::State;
350
351    /// Checks state compatibility without mutation.
352    fn validate_snapshot(&self, state: &Self::State) -> Result<()>;
353
354    /// Restores state accepted by [`Self::validate_snapshot`].
355    fn restore_validated(&mut self, state: &Self::State);
356}
357
358/// Checkpoint contract for one transform instance.
359///
360/// Serial checkpoints capture one instance; worker barriers restore each
361/// deterministic lane to its first unconsumed task.
362pub trait WorkerCheckpoint {
363    /// Owned serializable transform state.
364    type State: Clone + Serialize + DeserializeOwned;
365
366    /// Captures current transform state.
367    fn snapshot(&self) -> Self::State;
368
369    /// Checks state compatibility without mutation.
370    fn validate_snapshot(&self, state: &Self::State) -> Result<()>;
371
372    /// Restores state accepted by [`Self::validate_snapshot`].
373    fn restore_validated(&mut self, state: &Self::State);
374}
375
376/// Explicit unit-state adapter for a transform the caller asserts is stateless.
377///
378/// Construction is the opt-in assertion: mutable behavior that affects future
379/// outputs violates this adapter's exact-resume contract. This makes opaque
380/// closure transforms usable without requiring downstream orphan impls.
381#[derive(Clone, Copy, Debug, Default)]
382pub struct StatelessWorker<T> {
383    inner: T,
384}
385
386impl<T> StatelessWorker<T> {
387    /// Explicitly asserts that a transform has no state affecting replay.
388    pub fn new(inner: T) -> Self {
389        Self { inner }
390    }
391
392    /// Returns the wrapped transform.
393    pub fn into_inner(self) -> T {
394        self.inner
395    }
396}
397
398impl<Input, T> Transform<Input> for StatelessWorker<T>
399where
400    T: Transform<Input>,
401{
402    type Output = T::Output;
403    type Error = T::Error;
404
405    fn transform(
406        &mut self,
407        input: Input,
408        context: &TaskContext,
409    ) -> std::result::Result<Self::Output, Self::Error> {
410        self.inner.transform(input, context)
411    }
412}
413
414impl<T> WorkerCheckpoint for StatelessWorker<T> {
415    type State = ();
416
417    fn snapshot(&self) -> Self::State {}
418
419    fn validate_snapshot(&self, _state: &Self::State) -> Result<()> {
420        Ok(())
421    }
422
423    fn restore_validated(&mut self, _state: &Self::State) {}
424}
425
426/// Explicit transactional adapter for a stateful transform.
427#[derive(Clone, Copy, Debug, Default)]
428pub struct TransactionalWorker<T> {
429    inner: T,
430}
431
432impl<T> TransactionalWorker<T> {
433    /// Wraps a transform with transactional checkpoint support.
434    pub fn new(inner: T) -> Self {
435        Self { inner }
436    }
437
438    /// Returns the wrapped transform.
439    pub fn into_inner(self) -> T {
440        self.inner
441    }
442}
443
444impl<Input, T> Transform<Input> for TransactionalWorker<T>
445where
446    T: Transform<Input> + TransactionalCheckpoint,
447{
448    type Output = T::Output;
449    type Error = T::Error;
450
451    fn transform(
452        &mut self,
453        input: Input,
454        context: &TaskContext,
455    ) -> std::result::Result<Self::Output, Self::Error> {
456        self.inner.transform(input, context)
457    }
458}
459
460impl<T> WorkerCheckpoint for TransactionalWorker<T>
461where
462    T: TransactionalCheckpoint,
463{
464    type State = T::State;
465
466    fn snapshot(&self) -> Self::State {
467        self.inner.snapshot()
468    }
469
470    fn validate_snapshot(&self, state: &Self::State) -> Result<()> {
471        self.inner.validate_snapshot(state)
472    }
473
474    fn restore_validated(&mut self, state: &Self::State) {
475        self.inner.restore_validated(state);
476    }
477}
478
479#[doc(hidden)]
480#[derive(Clone, Copy, Debug)]
481pub struct StatelessTransformFactory<F>(pub(crate) F);
482
483impl<Input, F> TransformFactory<Input> for StatelessTransformFactory<F>
484where
485    F: TransformFactory<Input>,
486{
487    type Transform = StatelessWorker<F::Transform>;
488    type Error = F::Error;
489
490    fn create(
491        &self,
492        worker: Option<&WorkerContext>,
493    ) -> std::result::Result<Self::Transform, Self::Error> {
494        self.0.create(worker).map(StatelessWorker::new)
495    }
496}
497
498#[doc(hidden)]
499#[derive(Clone, Copy, Debug)]
500pub struct TransactionalTransformFactory<F>(pub(crate) F);
501
502impl<Input, F> TransformFactory<Input> for TransactionalTransformFactory<F>
503where
504    F: TransformFactory<Input>,
505    F::Transform: TransactionalCheckpoint,
506{
507    type Transform = TransactionalWorker<F::Transform>;
508    type Error = F::Error;
509
510    fn create(
511        &self,
512        worker: Option<&WorkerContext>,
513    ) -> std::result::Result<Self::Transform, Self::Error> {
514        self.0.create(worker).map(TransactionalWorker::new)
515    }
516}
517
518/// Resume-build error preserving either a RustTorch validation error or the
519/// transform factory's original error value.
520#[derive(Debug)]
521pub enum CheckpointBuildError<F> {
522    /// Loader configuration or component validation failed.
523    Configuration(RustTorchError),
524    /// Constructing the one retained serial transform failed.
525    TransformFactory(F),
526}
527
528impl<F> From<RustTorchError> for CheckpointBuildError<F> {
529    fn from(error: RustTorchError) -> Self {
530        Self::Configuration(error)
531    }
532}
533
534impl<F> fmt::Display for CheckpointBuildError<F>
535where
536    F: fmt::Display,
537{
538    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
539        match self {
540            Self::Configuration(error) => error.fmt(formatter),
541            Self::TransformFactory(error) => {
542                write!(
543                    formatter,
544                    "transform initialization failed while resuming: {error}"
545                )
546            }
547        }
548    }
549}
550
551impl<F> Error for CheckpointBuildError<F>
552where
553    F: Error + 'static,
554{
555    fn source(&self) -> Option<&(dyn Error + 'static)> {
556        match self {
557            Self::Configuration(error) => Some(error),
558            Self::TransformFactory(error) => Some(error),
559        }
560    }
561}
562
563/// Default builder/loader state with exact checkpointing disabled.
564#[derive(Clone, Copy, Debug, Default)]
565pub struct CheckpointDisabled;
566
567/// Builder state after a dataset identity enables a fresh exact checkpoint.
568///
569/// Exact checkpointing requires explicit replay evidence; an arbitrary
570/// dataset is not enabled merely by assigning an identity:
571///
572/// ```compile_fail
573/// use std::convert::Infallible;
574/// use rusttorch_data::{DataLoader, Dataset, VecCollate};
575/// struct Rows;
576/// impl Dataset for Rows {
577///     type Sample = i64;
578///     type Error = Infallible;
579///     fn len(&self) -> usize { 1 }
580///     fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
581/// }
582/// let _ = DataLoader::builder(Rows)
583///     .collate(VecCollate)
584///     .dataset_identity("rows".to_owned())
585///     .build();
586/// ```
587///
588/// Ordinary [`crate::TensorDataset`] rows share storage and are therefore not
589/// exact-checkpoint capable until [`crate::TensorDataset::into_replay_safe`]
590/// succeeds:
591///
592/// ```compile_fail
593/// use rusttorch_core::Tensor;
594/// use rusttorch_data::{DataLoader, TensorDataset, VecCollate};
595/// let dataset = TensorDataset::new(vec![Tensor::from_slice(&[1_i64])]).unwrap();
596/// let _ = DataLoader::builder(dataset)
597///     .collate(VecCollate)
598///     .dataset_identity("tensor".to_owned())
599///     .build();
600/// ```
601///
602/// An opaque closure transform has no restorable state contract unless the
603/// caller explicitly selects a stateless or transactional adapter:
604///
605/// ```compile_fail
606/// use std::convert::Infallible;
607/// use rusttorch_data::{
608///     DataLoader, Dataset, FnTransform, ReplaySafeDataset, ReplaySafeMap,
609///     TaskContext, VecCollate,
610/// };
611/// struct Rows;
612/// impl Dataset for Rows {
613///     type Sample = i64;
614///     type Error = Infallible;
615///     fn len(&self) -> usize { 1 }
616///     fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
617/// }
618/// impl ReplaySafeDataset for Rows {}
619/// let transform = FnTransform::new(|value, _: &TaskContext| Ok::<_, Infallible>(value));
620/// let _ = DataLoader::builder(ReplaySafeMap::new(Rows))
621///     .transform(transform)
622///     .collate(VecCollate)
623///     .dataset_identity("rows".to_owned())
624///     .build();
625/// ```
626///
627/// Closure-backed collation is likewise opaque without an explicit
628/// [`Checkpointable`] implementation:
629///
630/// ```compile_fail
631/// use std::convert::Infallible;
632/// use rusttorch_data::{
633///     DataLoader, Dataset, FnCollate, ReplaySafeDataset, ReplaySafeMap,
634/// };
635/// struct Rows;
636/// impl Dataset for Rows {
637///     type Sample = i64;
638///     type Error = Infallible;
639///     fn len(&self) -> usize { 1 }
640///     fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
641/// }
642/// impl ReplaySafeDataset for Rows {}
643/// let _ = DataLoader::builder(ReplaySafeMap::new(Rows))
644///     .collate(FnCollate::new(|values| Ok::<_, Infallible>(values)))
645///     .dataset_identity("rows".to_owned())
646///     .build();
647/// ```
648///
649/// Closure-backed sampler and explicit-batch sources do not expose stable
650/// configuration/cursor state:
651///
652/// ```compile_fail
653/// use std::convert::Infallible;
654/// use rusttorch_data::{
655///     DataLoader, Dataset, FnSampler, ReplaySafeDataset, ReplaySafeMap,
656///     VecCollate,
657/// };
658/// struct Rows;
659/// impl Dataset for Rows {
660///     type Sample = i64;
661///     type Error = Infallible;
662///     fn len(&self) -> usize { 1 }
663///     fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
664/// }
665/// impl ReplaySafeDataset for Rows {}
666/// let _ = DataLoader::builder(ReplaySafeMap::new(Rows))
667///     .sampler(FnSampler::new(Some(1), |_| [0].into_iter()))
668///     .collate(VecCollate)
669///     .dataset_identity("rows".to_owned())
670///     .build();
671/// ```
672///
673/// ```compile_fail
674/// use std::convert::Infallible;
675/// use rusttorch_data::{
676///     DataLoader, Dataset, FnBatchSource, ReplaySafeDataset, ReplaySafeMap,
677///     VecCollate,
678/// };
679/// struct Rows;
680/// impl Dataset for Rows {
681///     type Sample = i64;
682///     type Error = Infallible;
683///     fn len(&self) -> usize { 1 }
684///     fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
685/// }
686/// impl ReplaySafeDataset for Rows {}
687/// let _ = DataLoader::builder(ReplaySafeMap::new(Rows))
688///     .batch_sampler(FnBatchSource::new(Some(1), |_| vec![vec![0]].into_iter()))
689///     .collate(VecCollate)
690///     .dataset_identity("rows".to_owned())
691///     .build();
692/// ```
693///
694/// Worker replay supports explicit immutable datasets and checkpointable
695/// transforms. Zero workers use the same tagged envelope:
696///
697/// ```
698/// use std::convert::Infallible;
699/// use rusttorch_data::{
700///     DataLoader, Dataset, ReplaySafeDataset, ReplaySafeMap, VecCollate,
701/// };
702/// struct Rows;
703/// impl Dataset for Rows {
704///     type Sample = i64;
705///     type Error = Infallible;
706///     fn len(&self) -> usize { 1 }
707///     fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
708/// }
709/// impl ReplaySafeDataset for Rows {}
710/// let mut loader = DataLoader::builder(ReplaySafeMap::new(Rows))
711///     .workers(0)
712///     .collate(VecCollate)
713///     .dataset_identity("rows".to_owned())
714///     .build().unwrap();
715/// let state = loader.iter().checkpoint().unwrap();
716/// let mut resumed = DataLoader::builder(ReplaySafeMap::new(Rows))
717///     .workers(0).collate(VecCollate).dataset_identity("rows".to_owned())
718///     .resume_from(state).build().unwrap();
719/// assert_eq!(resumed.iter().next().unwrap().unwrap(), vec![1]);
720/// ```
721#[derive(Clone, Debug)]
722pub struct CheckpointFresh {
723    pub(crate) identity: String,
724}
725
726/// Builder state carrying a typed checkpoint selected for resume.
727#[derive(Clone, Debug)]
728pub struct CheckpointResume<S> {
729    pub(crate) identity: String,
730    pub(crate) state: S,
731}
732
733/// Retained first serial cursor and transform for a resumed loader.
734#[doc(hidden)]
735pub struct PendingSerial<I, T> {
736    pub(crate) batches: I,
737    pub(crate) transform: T,
738    pub(crate) next_batch: u64,
739    pub(crate) next_logical_sample: u64,
740}
741
742/// Built-loader state enabling exact serial checkpoints.
743#[doc(hidden)]
744pub struct CheckpointActive<I, T> {
745    pub(crate) identity: String,
746    pub(crate) configuration: LoaderConfiguration,
747    pub(crate) pending: Option<PendingSerial<I, T>>,
748}
749
750/// Active-iterator state at an exact serial checkpoint boundary.
751#[doc(hidden)]
752pub struct CheckpointIteration {
753    pub(crate) identity: String,
754    pub(crate) configuration: LoaderConfiguration,
755    pub(crate) boundary_valid: bool,
756}
757
758/// One deterministic worker lane's committed transform state.
759#[derive(Clone, Debug, Deserialize, Serialize)]
760pub struct WorkerLaneState<S> {
761    /// Zero-based lane identity.
762    pub id: usize,
763    /// Transform state before this lane's next unconsumed task.
764    pub state: S,
765}
766
767/// Explicit zero-worker or positive-worker transform state.
768#[derive(Clone, Debug, Deserialize, Serialize)]
769pub enum WorkerTransformLanes<S> {
770    /// One transform running on the calling thread.
771    Serial(S),
772    /// Transform states in deterministic lane order.
773    Workers(Vec<WorkerLaneState<S>>),
774}
775
776/// Versioned worker replay envelope within schema-v1 loader state.
777///
778/// An opaque transform cannot enable worker checkpointing without an explicit
779/// adapter, even when its closure appears deterministic:
780///
781/// ```compile_fail
782/// use std::convert::Infallible;
783/// use rusttorch_data::{DataLoader, Dataset, FnTransform, ReplaySafeDataset, ReplaySafeMap, TaskContext, VecCollate};
784/// struct Rows;
785/// impl Dataset for Rows {
786///     type Sample = i64;
787///     type Error = Infallible;
788///     fn len(&self) -> usize { 1 }
789///     fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
790/// }
791/// impl ReplaySafeDataset for Rows {}
792/// let _ = DataLoader::builder(ReplaySafeMap::new(Rows))
793///     .workers(2).collate(VecCollate)
794///     .transform(FnTransform::new(|x: i64, _: &TaskContext| Ok::<_, Infallible>(x)))
795///     .dataset_identity("rows".to_owned()).build();
796/// ```
797#[derive(Clone, Debug, Deserialize, Serialize)]
798pub struct WorkerTransformState<S> {
799    /// Worker envelope version, currently one.
800    pub schema_version: u32,
801    /// Exact worker count.
802    pub workers: usize,
803    /// Effective prefetch factor; absent for zero workers.
804    pub prefetch_factor: Option<usize>,
805    /// Original transform factory initialization identity.
806    pub factory_generation: u64,
807    /// Current transport generation used to reject stale messages.
808    pub run_generation: u64,
809    /// Explicit lane identities and states.
810    pub lanes: WorkerTransformLanes<S>,
811}
812
813/// Retained worker checkpoint components for the next exact iteration.
814#[doc(hidden)]
815pub struct WorkerCheckpointActive<I, T, S> {
816    pub(crate) identity: String,
817    pub(crate) configuration: LoaderConfiguration,
818    pub(crate) batches: Option<I>,
819    pub(crate) serial: Option<T>,
820    pub(crate) barrier: Option<crate::worker_checkpoint::Barrier<S>>,
821    pub(crate) next_batch: u64,
822    pub(crate) next_logical_sample: u64,
823    pub(crate) factory_generation: u64,
824    pub(crate) run_generation: u64,
825}
826
827/// Committed coordinator and worker boundary for one exact iteration.
828#[doc(hidden)]
829pub struct WorkerCheckpointIteration<S, E> {
830    pub(crate) identity: String,
831    pub(crate) configuration: LoaderConfiguration,
832    pub(crate) barrier: Option<crate::worker_checkpoint::Barrier<S>>,
833    pub(crate) logical_ends: std::collections::BTreeMap<u64, u64>,
834    pub(crate) errors: std::collections::BTreeMap<u64, E>,
835    pub(crate) next_logical_sample: u64,
836    pub(crate) factory_generation: u64,
837    pub(crate) boundary_valid: bool,
838}
839
840impl Checkpointable for () {
841    type State = ();
842
843    fn save_state(&self) -> Self::State {}
844
845    fn validate_state(&self, _state: &Self::State) -> Result<()> {
846        Ok(())
847    }
848
849    fn load_validated(&mut self, _state: &Self::State) {}
850}