Skip to main content

rusttorch_data/
checkpoint.rs

1use std::{error::Error, fmt};
2
3use rusttorch_core::{Result, RustTorchError};
4use serde::{Deserialize, Serialize, de::DeserializeOwned};
5
6use crate::{TaskContext, Transform, TransformFactory, WorkerContext};
7
8/// A stream reader whose next unread position can be saved and restored exactly.
9///
10/// Implement this for seekable files, versioned record stores, or other sources
11/// where replaying a read produces the same record. The state must contain every
12/// cursor and decoder setting that affects later output; keep it owned and
13/// independent of mutable buffers. A destructive queue cannot promise this
14/// contract merely by remembering how many items were read.
15///
16/// [`crate::ExactStreamDataLoader`] demonstrates a lazy source, cursor validation,
17/// JSON state, and a restart that yields the same remaining batches.
18///
19/// Validation is read-only; applying accepted state must not fail or panic.
20/// Records must have strictly increasing per-shard global sequence IDs, and all
21/// shards together must cover the ordered global sequence without gaps. If the
22/// source retains a [`WorkerContext`], implement [`Self::set_run_context`] so a
23/// checkpoint barrier can replace the cancelled context before further reads.
24pub trait CheckpointableSource:
25    Iterator<Item = std::result::Result<crate::WorkerRecord<Self::Sample>, Self::Error>>
26{
27    /// Owned decoded sample.
28    type Sample;
29    /// Owned state, independent of future decoder progress.
30    type State: Clone + Serialize + DeserializeOwned;
31    /// Preserved source error.
32    type Error;
33    /// Returns the failed read's stable global position without changing state.
34    ///
35    /// Called after `next()` returns an error. This position must obey the
36    /// shard's strictly increasing sequence contract and repeat after rollback.
37    fn error_sequence(&self, error: &Self::Error) -> crate::SequenceId;
38    /// Captures the boundary before the next read attempt.
39    fn snapshot(&self) -> Self::State;
40    /// Validates without changing the source.
41    fn validate_snapshot(&self, state: &Self::State) -> std::result::Result<(), Self::Error>;
42    /// Applies state previously accepted by validation.
43    fn restore_validated(&mut self, state: &Self::State);
44    /// Replaces the run cancellation/deadline context before resumed reads.
45    ///
46    /// Sources retaining a context must override this method and replace it.
47    /// Logical worker generation and seed remain unchanged. This must not fail.
48    fn set_run_context(&mut self, _context: WorkerContext) {}
49}
50
51/// Stable factory identity for explicitly checkpointable stream sources.
52///
53/// Implement this alongside [`CheckpointableSource`] on the factory's source
54/// type. `CHECKPOINT_KIND` identifies the implementation and cursor format;
55/// `.checkpointable("contents-version")` separately identifies the actual data.
56/// Change the kind when a cursor format or its meaning changes, and change the
57/// contents identity when records change. See [`crate::ExactStreamDataLoader`]
58/// for a complete successful implementation.
59///
60/// An ordinary factory cannot acquire exact replay just by setting an identity:
61///
62/// ```compile_fail
63/// use rusttorch_data::*;
64/// use std::convert::Infallible;
65/// struct Factory;
66/// impl WorkerSourceFactory for Factory {
67///     type Sample = usize;
68///     type Error = Infallible;
69///     type Source = std::iter::Empty<Result<WorkerRecord<usize>, Infallible>>;
70///     fn create(&self, _: WorkerContext) -> Result<Self::Source, Infallible> { Ok(std::iter::empty()) }
71/// }
72/// StreamDataLoaderBuilder::new(Factory).collate(VecCollate)
73///     .checkpointable("ordinary").build().unwrap();
74/// ```
75///
76/// Opaque transforms also need an explicit `WorkerCheckpoint` implementation or
77/// an explicit `StatelessWorker`/`TransactionalWorker` adapter:
78///
79/// ```compile_fail
80/// use rusttorch_data::*;
81/// use std::convert::Infallible;
82/// struct Source;
83/// impl Iterator for Source {
84///     type Item = Result<WorkerRecord<usize>, Infallible>;
85///     fn next(&mut self) -> Option<Self::Item> { None }
86/// }
87/// impl CheckpointableSource for Source {
88///     type Sample = usize;
89///     type Error = Infallible;
90///     type State = ();
91///     fn error_sequence(&self, error: &Infallible) -> SequenceId { match *error {} }
92///     fn snapshot(&self) {}
93///     fn validate_snapshot(&self, _: &()) -> Result<(), Infallible> { Ok(()) }
94///     fn restore_validated(&mut self, _: &()) {}
95/// }
96/// struct Factory;
97/// impl WorkerSourceFactory for Factory {
98///     type Sample = usize;
99///     type Error = Infallible;
100///     type Source = Source;
101///     fn create(&self, _: WorkerContext) -> Result<Source, Infallible> { Ok(Source) }
102/// }
103/// impl CheckpointSourceFactory for Factory { const CHECKPOINT_KIND: &'static str = "empty.v1"; }
104/// let transform = FnTransform::new(|sample: usize, _: &TaskContext| Ok::<_, Infallible>(sample));
105/// StreamDataLoaderBuilder::new(Factory).transform(transform).collate(VecCollate)
106///     .checkpointable("opaque").build().unwrap();
107/// ```
108pub trait CheckpointSourceFactory: crate::WorkerSourceFactory {
109    /// Stable source kind and state-format version, for example `records.v1`.
110    const CHECKPOINT_KIND: &'static str;
111}
112
113/// Exact streaming configuration, with no sampler state.
114#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
115pub struct StreamCheckpointConfiguration {
116    /// Positive shard count.
117    pub workers: usize,
118    /// Maximum unpublished records per shard.
119    pub prefetch_factor: usize,
120    /// Coordinator batch size.
121    pub batch_size: usize,
122    /// Global short-tail policy.
123    pub drop_last: bool,
124    /// Task randomness seed.
125    pub loader_seed: u64,
126    /// Distributed task rank.
127    pub rank: usize,
128    /// Requested pinning behavior.
129    pub pin_request: CheckpointPinRequest,
130    /// Effective pinning behavior.
131    pub pin_status: CheckpointPinStatus,
132}
133
134/// Paired source and transform boundary for one worker shard.
135#[derive(Clone, Debug, Deserialize, Serialize)]
136pub struct StreamLaneState<S, T> {
137    /// Unique worker index in canonical order.
138    pub id: usize,
139    /// Source state preceding its first unconsumed attempt.
140    pub source: S,
141    /// Transform state preceding the same attempt.
142    pub transform: T,
143}
144
145/// Serializable input position and component state for resuming a sharded stream.
146///
147/// Obtain this from [`crate::ExactStreamLoaderIter::checkpoint`] and pass the
148/// decoded value to [`crate::StreamDataLoaderBuilder::resume`]. Store it with the
149/// model, optimizer and training counters from the same completed batch. It
150/// contains shard cursors and transform/collator state, not decoded records or
151/// model tensors. [`crate::ExactStreamDataLoader`] demonstrates JSON round-trip
152/// and exact replay; applications choose their storage format and atomic writes.
153#[derive(Clone, Debug, Deserialize, Serialize)]
154pub struct StreamLoaderState<S, T = (), C = ()> {
155    /// Stream schema version; currently one.
156    pub schema_version: u32,
157    /// Caller identity for exact source contents.
158    pub source_identity: String,
159    /// Stable factory kind/version.
160    pub factory_kind: String,
161    /// Task epoch.
162    pub epoch: u64,
163    /// Logical source and transform initialization generation.
164    pub source_generation: u64,
165    /// Transport generation, advanced by each checkpoint barrier.
166    pub transport_generation: u64,
167    /// Next visible batch index.
168    pub next_batch: u64,
169    /// Next expected global sequence ID.
170    pub next_sequence: u64,
171    /// Whether the last visible batch was the final incomplete global tail.
172    pub short_tail: bool,
173    /// Exact global source length when supplied by the factory.
174    pub source_length: Option<usize>,
175    /// Task RNG derivation version.
176    pub rng_derivation_version: u32,
177    /// Worker seed derivation version.
178    pub worker_seed_derivation_version: u32,
179    /// Exact runtime settings.
180    pub configuration: StreamCheckpointConfiguration,
181    /// Canonically ordered worker states; never contains decoded samples.
182    pub lanes: Vec<StreamLaneState<S, T>>,
183    /// Coordinator collation state.
184    pub coordinator: C,
185}
186
187/// Typed stream construction or transactional resume failure.
188#[derive(Debug, thiserror::Error)]
189pub enum StreamCheckpointBuildError<S, F> {
190    /// Static configuration rejection.
191    #[error(transparent)]
192    Configuration(#[from] RustTorchError),
193    /// Source creation or read-only state validation failed.
194    #[error("stream source on worker {worker} failed: {source}")]
195    Source {
196        /// Worker lane.
197        worker: usize,
198        /// Original source error.
199        #[source]
200        source: S,
201    },
202    /// Transform creation failed.
203    #[error("stream transform initialization on worker {worker} failed: {source}")]
204    TransformFactory {
205        /// Worker lane.
206        worker: usize,
207        /// Original factory error.
208        #[source]
209        source: F,
210    },
211    /// Transform state validation failed.
212    #[error("stream transform validation on worker {worker} failed: {source}")]
213    TransformState {
214        /// Worker lane.
215        worker: usize,
216        /// Original validation error.
217        #[source]
218        source: RustTorchError,
219    },
220    /// Initialization or checkpoint callback panicked.
221    #[error("stream checkpoint worker {worker} panicked")]
222    WorkerPanic {
223        /// Worker lane.
224        worker: usize,
225    },
226}
227
228/// Schema version emitted by serial map-loader checkpoints.
229pub const LOADER_STATE_SCHEMA_VERSION: u32 = 1;
230
231/// Pinning request recorded in a loader checkpoint.
232#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
233pub enum CheckpointPinRequest {
234    /// Pinning was not requested.
235    Disabled,
236    /// Pinning was requested only when a CUDA accelerator is available.
237    Auto,
238    /// Pinning was requested for one CUDA device.
239    ExplicitCuda {
240        /// CUDA device index.
241        index: usize,
242    },
243}
244
245/// Effective pinning behavior recorded in a loader checkpoint.
246#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
247pub enum CheckpointPinStatus {
248    /// Pinning was not requested.
249    Disabled,
250    /// Automatic pinning found no supported accelerator.
251    DisabledNoAccelerator,
252    /// Batches are pinned for one CUDA device.
253    Cuda {
254        /// CUDA device index.
255        index: usize,
256    },
257}
258
259/// Loader settings that must match when a serial checkpoint is resumed.
260#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
261pub struct LoaderConfiguration {
262    /// Automatic batch size, or `None` for explicit/no batching.
263    pub batch_size: Option<usize>,
264    /// Whether an incomplete automatic batch is omitted.
265    pub drop_last: bool,
266    /// Exact worker count; zero for serial execution.
267    pub workers: usize,
268    /// Effective worker prefetch factor, or `None` for serial execution.
269    pub prefetch_factor: Option<usize>,
270    /// Whether results follow sampler order.
271    pub in_order: bool,
272    /// Loader seed used by task-local random transforms.
273    pub loader_seed: u64,
274    /// Distributed task rank.
275    pub rank: usize,
276    /// Distributed replica count, or one for an unsharded sampler.
277    pub replicas: usize,
278    /// Distributed shuffle policy when applicable.
279    pub distributed_shuffle: Option<bool>,
280    /// Distributed sampler seed when applicable.
281    pub distributed_seed: Option<u64>,
282    /// Distributed tail policy when applicable.
283    pub distributed_drop_last: Option<bool>,
284    /// Stable RustTorch sampler/plan kind.
285    pub sampler_kind: String,
286    /// Requested pinning policy.
287    pub pin_request: CheckpointPinRequest,
288    /// Effective pinning status visible to consumers.
289    pub pin_status: CheckpointPinStatus,
290}
291
292/// Serializable input position and pipeline state for an exact map-loader restart.
293///
294/// Capture this on the iterator, after the application has successfully used a
295/// batch. Recreate the same dataset and options, deserialize the state, and pass
296/// it to [`crate::DataLoaderBuilder::resume_from`]. The next iterator resumes
297/// before the first unconsumed batch. Prefetched samples are read again from
298/// saved component state; sample payloads are not serialized.
299///
300/// # Save and restore a prefetched loader
301///
302/// The immutable rows below can safely be fetched again. A content version
303/// identifies the data, while the loader validates sampling, batching, worker
304/// and component configuration during restore. This example uses `serde_json`;
305/// add it as a dependency when choosing JSON in your application.
306///
307/// ```
308/// use std::convert::Infallible;
309/// use rusttorch_data::{DataLoader, Dataset, ReplaySafeDataset, ReplaySafeMap, VecCollate};
310/// struct Rows([i64; 5]);
311/// impl Dataset for Rows {
312///     type Sample = i64;
313///     type Error = Infallible;
314///     fn len(&self) -> usize { self.0.len() }
315///     fn get(&self, index: usize) -> Result<i64, Infallible> { Ok(self.0[index]) }
316/// }
317/// impl ReplaySafeDataset for Rows {}
318///
319/// fn main() -> Result<(), Box<dyn std::error::Error>> {
320///     let build = || DataLoader::builder(ReplaySafeMap::new(Rows([2, 3, 5, 7, 11])))
321///         .batch_size(2).workers(2).prefetch_factor(2).collate(VecCollate)
322///         .dataset_identity("prime-rows-v1".to_owned());
323///     let mut loader = build().build()?;
324///     let mut iteration = loader.iter();
325///     assert_eq!(iteration.next().unwrap()?, [2, 3]);
326///     let saved = serde_json::to_string(&iteration.checkpoint()?)?;
327///     let uninterrupted = iteration.collect::<Result<Vec<_>, _>>()?;
328///
329///     let mut restored = build().resume_from(serde_json::from_str(&saved)?).build()?;
330///     let remaining = restored.iter().collect::<Result<Vec<_>, _>>()?;
331///     assert_eq!(remaining, [vec![5, 7], vec![11]]);
332///     assert_eq!(remaining, uninterrupted);
333///     Ok(())
334/// }
335/// ```
336///
337/// Save model and optimizer state at the same training step; this state covers
338/// only the input pipeline. Capture initially or after a successful batch, before
339/// observing an error or exhaustion. Do not change the selected epoch before
340/// consuming a pending resume: `set_epoch` intentionally starts a fresh pass.
341///
342/// Serial checkpoints support replay-safe or transactional datasets. Worker
343/// checkpoints require replay-safe datasets and ordered delivery, with no byte
344/// budget, timeout, custom worker initializer or persistent workers. Custom
345/// transforms and collation need explicit checkpoint contracts. Use
346/// [`crate::TensorDataset::into_replay_safe`] for isolated serial tensor data.
347#[derive(Clone, Debug, Deserialize, Serialize)]
348pub struct LoaderState<D, S, T = (), C = ()> {
349    /// State schema version.
350    pub schema_version: u32,
351    /// Caller-defined identity for the exact dataset contents.
352    pub dataset_identity: String,
353    /// Active sampler epoch.
354    pub epoch: u64,
355    /// Transport generation. Serial state is always zero.
356    pub iterator_generation: u64,
357    /// Next consumer-visible batch number.
358    pub next_batch: u64,
359    /// Number of logical sample occurrences already consumed.
360    pub next_logical_sample: u64,
361    /// Dataset component state.
362    pub dataset: D,
363    /// Active sampler or batch-source cursor state.
364    pub sampler: S,
365    /// Serial transform state or a versioned worker transform envelope.
366    pub transform: T,
367    /// Effective coordinator collator/converter state.
368    pub collate: C,
369    /// Task-RNG derivation version.
370    pub rng_derivation_version: u32,
371    /// Worker-seed derivation version.
372    pub worker_seed_derivation_version: u32,
373    /// Exact loader configuration identity.
374    pub configuration: LoaderConfiguration,
375}
376
377/// A component whose state can be validated before it is restored.
378///
379/// Implement this on a custom collator or converter when it retains state that
380/// affects later batches. Built-in collators already supply it. Capture all
381/// relevant counters/configuration in an owned serde state; validation should
382/// reject incompatible values without resetting the live component.
383///
384/// Validation must be read-only. `load_validated` must not fail or panic after
385/// the same component accepted the supplied state; violating either rule
386/// breaks the resume transaction contract.
387pub trait Checkpointable {
388    /// Owned serializable component state.
389    type State: Clone + Serialize + DeserializeOwned;
390
391    /// Captures the current component state.
392    fn save_state(&self) -> Self::State;
393
394    /// Checks state compatibility without mutating the component.
395    fn validate_state(&self, state: &Self::State) -> Result<()>;
396
397    /// Restores state previously accepted by [`Self::validate_state`].
398    fn load_validated(&mut self, state: &Self::State);
399}
400
401/// Promise that fetching an index again from fixed dataset contents is safe.
402///
403/// Implement this only when reads have no state changes that affect later
404/// samples and no external side effects that must happen exactly once. Wrap the
405/// dataset in [`crate::ReplaySafeMap`] to use this contract in an exact loader;
406/// [`LoaderState`] includes a complete example. Mutable/stateful datasets should
407/// use [`DatasetCheckpoint`] through [`crate::TransactionalMap`] instead.
408pub trait ReplaySafeDataset: crate::Dataset {}
409
410/// Dataset state used by exact serial loader checkpoints.
411///
412/// Validation must be read-only, and restore must be infallible after matching
413/// validation succeeds.
414pub trait DatasetCheckpoint: crate::Dataset {
415    /// Owned serializable dataset state.
416    type State: Clone + Serialize + DeserializeOwned;
417
418    /// Captures dataset state at the visible boundary.
419    fn snapshot_dataset(&self) -> Self::State;
420
421    /// Checks state compatibility without mutation.
422    fn validate_dataset_state(&self, state: &Self::State) -> Result<()>;
423
424    /// Restores state accepted by [`Self::validate_dataset_state`].
425    fn restore_dataset_validated(&mut self, state: &Self::State);
426}
427
428/// Marker for a component with no mutable checkpoint state.
429pub trait Stateless {}
430
431/// Transactional state contract for a serial component.
432///
433/// Validation must be read-only, and restore must be infallible after matching
434/// validation succeeds.
435pub trait TransactionalCheckpoint {
436    /// Owned serializable component state.
437    type State: Clone + Serialize + DeserializeOwned;
438
439    /// Captures current component state.
440    fn snapshot(&self) -> Self::State;
441
442    /// Checks state compatibility without mutation.
443    fn validate_snapshot(&self, state: &Self::State) -> Result<()>;
444
445    /// Restores state accepted by [`Self::validate_snapshot`].
446    fn restore_validated(&mut self, state: &Self::State);
447}
448
449/// Checkpoint contract for one transform instance.
450///
451/// Serial checkpoints capture one instance; worker barriers restore each
452/// deterministic lane to its first unconsumed task.
453pub trait WorkerCheckpoint {
454    /// Owned serializable transform state.
455    type State: Clone + Serialize + DeserializeOwned;
456
457    /// Captures current transform state.
458    fn snapshot(&self) -> Self::State;
459
460    /// Checks state compatibility without mutation.
461    fn validate_snapshot(&self, state: &Self::State) -> Result<()>;
462
463    /// Restores state accepted by [`Self::validate_snapshot`].
464    fn restore_validated(&mut self, state: &Self::State);
465}
466
467/// Explicit unit-state adapter for a transform the caller asserts is stateless.
468///
469/// Construction is the opt-in assertion: mutable behavior that affects future
470/// outputs violates this adapter's exact-resume contract. This makes opaque
471/// closure transforms usable without requiring downstream orphan impls.
472#[derive(Clone, Copy, Debug, Default)]
473pub struct StatelessWorker<T> {
474    inner: T,
475}
476
477impl<T> StatelessWorker<T> {
478    /// Explicitly asserts that a transform has no state affecting replay.
479    pub fn new(inner: T) -> Self {
480        Self { inner }
481    }
482
483    /// Returns the wrapped transform.
484    pub fn into_inner(self) -> T {
485        self.inner
486    }
487}
488
489impl<Input, T> Transform<Input> for StatelessWorker<T>
490where
491    T: Transform<Input>,
492{
493    type Output = T::Output;
494    type Error = T::Error;
495
496    fn transform(
497        &mut self,
498        input: Input,
499        context: &TaskContext,
500    ) -> std::result::Result<Self::Output, Self::Error> {
501        self.inner.transform(input, context)
502    }
503}
504
505impl<T> WorkerCheckpoint for StatelessWorker<T> {
506    type State = ();
507
508    fn snapshot(&self) -> Self::State {}
509
510    fn validate_snapshot(&self, _state: &Self::State) -> Result<()> {
511        Ok(())
512    }
513
514    fn restore_validated(&mut self, _state: &Self::State) {}
515}
516
517/// Explicit transactional adapter for a stateful transform.
518#[derive(Clone, Copy, Debug, Default)]
519pub struct TransactionalWorker<T> {
520    inner: T,
521}
522
523impl<T> TransactionalWorker<T> {
524    /// Wraps a transform with transactional checkpoint support.
525    pub fn new(inner: T) -> Self {
526        Self { inner }
527    }
528
529    /// Returns the wrapped transform.
530    pub fn into_inner(self) -> T {
531        self.inner
532    }
533}
534
535impl<Input, T> Transform<Input> for TransactionalWorker<T>
536where
537    T: Transform<Input> + TransactionalCheckpoint,
538{
539    type Output = T::Output;
540    type Error = T::Error;
541
542    fn transform(
543        &mut self,
544        input: Input,
545        context: &TaskContext,
546    ) -> std::result::Result<Self::Output, Self::Error> {
547        self.inner.transform(input, context)
548    }
549}
550
551impl<T> WorkerCheckpoint for TransactionalWorker<T>
552where
553    T: TransactionalCheckpoint,
554{
555    type State = T::State;
556
557    fn snapshot(&self) -> Self::State {
558        self.inner.snapshot()
559    }
560
561    fn validate_snapshot(&self, state: &Self::State) -> Result<()> {
562        self.inner.validate_snapshot(state)
563    }
564
565    fn restore_validated(&mut self, state: &Self::State) {
566        self.inner.restore_validated(state);
567    }
568}
569
570#[doc(hidden)]
571#[derive(Clone, Copy, Debug)]
572pub struct StatelessTransformFactory<F>(pub(crate) F);
573
574impl<Input, F> TransformFactory<Input> for StatelessTransformFactory<F>
575where
576    F: TransformFactory<Input>,
577{
578    type Transform = StatelessWorker<F::Transform>;
579    type Error = F::Error;
580
581    fn create(
582        &self,
583        worker: Option<&WorkerContext>,
584    ) -> std::result::Result<Self::Transform, Self::Error> {
585        self.0.create(worker).map(StatelessWorker::new)
586    }
587}
588
589#[doc(hidden)]
590#[derive(Clone, Copy, Debug)]
591pub struct TransactionalTransformFactory<F>(pub(crate) F);
592
593impl<Input, F> TransformFactory<Input> for TransactionalTransformFactory<F>
594where
595    F: TransformFactory<Input>,
596    F::Transform: TransactionalCheckpoint,
597{
598    type Transform = TransactionalWorker<F::Transform>;
599    type Error = F::Error;
600
601    fn create(
602        &self,
603        worker: Option<&WorkerContext>,
604    ) -> std::result::Result<Self::Transform, Self::Error> {
605        self.0.create(worker).map(TransactionalWorker::new)
606    }
607}
608
609/// Resume-build error preserving either a RustTorch validation error or the
610/// transform factory's original error value.
611#[derive(Debug)]
612pub enum CheckpointBuildError<F> {
613    /// Loader configuration or component validation failed.
614    Configuration(RustTorchError),
615    /// Constructing the one retained serial transform failed.
616    TransformFactory(F),
617}
618
619impl<F> From<RustTorchError> for CheckpointBuildError<F> {
620    fn from(error: RustTorchError) -> Self {
621        Self::Configuration(error)
622    }
623}
624
625impl<F> fmt::Display for CheckpointBuildError<F>
626where
627    F: fmt::Display,
628{
629    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
630        match self {
631            Self::Configuration(error) => error.fmt(formatter),
632            Self::TransformFactory(error) => {
633                write!(
634                    formatter,
635                    "transform initialization failed while resuming: {error}"
636                )
637            }
638        }
639    }
640}
641
642impl<F> Error for CheckpointBuildError<F>
643where
644    F: Error + 'static,
645{
646    fn source(&self) -> Option<&(dyn Error + 'static)> {
647        match self {
648            Self::Configuration(error) => Some(error),
649            Self::TransformFactory(error) => Some(error),
650        }
651    }
652}
653
654/// Default builder/loader state with exact checkpointing disabled.
655#[derive(Clone, Copy, Debug, Default)]
656pub struct CheckpointDisabled;
657
658/// Builder state after a dataset identity enables a fresh exact checkpoint.
659///
660/// Exact checkpointing requires explicit replay evidence; an arbitrary
661/// dataset is not enabled merely by assigning an identity:
662///
663/// ```compile_fail
664/// use std::convert::Infallible;
665/// use rusttorch_data::{DataLoader, Dataset, VecCollate};
666/// struct Rows;
667/// impl Dataset for Rows {
668///     type Sample = i64;
669///     type Error = Infallible;
670///     fn len(&self) -> usize { 1 }
671///     fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
672/// }
673/// let _ = DataLoader::builder(Rows)
674///     .collate(VecCollate)
675///     .dataset_identity("rows".to_owned())
676///     .build();
677/// ```
678///
679/// Ordinary [`crate::TensorDataset`] rows share storage and are therefore not
680/// exact-checkpoint capable until [`crate::TensorDataset::into_replay_safe`]
681/// succeeds:
682///
683/// ```compile_fail
684/// use rusttorch_core::Tensor;
685/// use rusttorch_data::{DataLoader, TensorDataset, VecCollate};
686/// let dataset = TensorDataset::new(vec![Tensor::from_slice(&[1_i64])]).unwrap();
687/// let _ = DataLoader::builder(dataset)
688///     .collate(VecCollate)
689///     .dataset_identity("tensor".to_owned())
690///     .build();
691/// ```
692///
693/// An opaque closure transform has no restorable state contract unless the
694/// caller explicitly selects a stateless or transactional adapter:
695///
696/// ```compile_fail
697/// use std::convert::Infallible;
698/// use rusttorch_data::{
699///     DataLoader, Dataset, FnTransform, ReplaySafeDataset, ReplaySafeMap,
700///     TaskContext, VecCollate,
701/// };
702/// struct Rows;
703/// impl Dataset for Rows {
704///     type Sample = i64;
705///     type Error = Infallible;
706///     fn len(&self) -> usize { 1 }
707///     fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
708/// }
709/// impl ReplaySafeDataset for Rows {}
710/// let transform = FnTransform::new(|value, _: &TaskContext| Ok::<_, Infallible>(value));
711/// let _ = DataLoader::builder(ReplaySafeMap::new(Rows))
712///     .transform(transform)
713///     .collate(VecCollate)
714///     .dataset_identity("rows".to_owned())
715///     .build();
716/// ```
717///
718/// Closure-backed collation is likewise opaque without an explicit
719/// [`Checkpointable`] implementation:
720///
721/// ```compile_fail
722/// use std::convert::Infallible;
723/// use rusttorch_data::{
724///     DataLoader, Dataset, FnCollate, ReplaySafeDataset, ReplaySafeMap,
725/// };
726/// struct Rows;
727/// impl Dataset for Rows {
728///     type Sample = i64;
729///     type Error = Infallible;
730///     fn len(&self) -> usize { 1 }
731///     fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
732/// }
733/// impl ReplaySafeDataset for Rows {}
734/// let _ = DataLoader::builder(ReplaySafeMap::new(Rows))
735///     .collate(FnCollate::new(|values| Ok::<_, Infallible>(values)))
736///     .dataset_identity("rows".to_owned())
737///     .build();
738/// ```
739///
740/// Closure-backed sampler and explicit-batch sources do not expose stable
741/// configuration/cursor state:
742///
743/// ```compile_fail
744/// use std::convert::Infallible;
745/// use rusttorch_data::{
746///     DataLoader, Dataset, FnSampler, ReplaySafeDataset, ReplaySafeMap,
747///     VecCollate,
748/// };
749/// struct Rows;
750/// impl Dataset for Rows {
751///     type Sample = i64;
752///     type Error = Infallible;
753///     fn len(&self) -> usize { 1 }
754///     fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
755/// }
756/// impl ReplaySafeDataset for Rows {}
757/// let _ = DataLoader::builder(ReplaySafeMap::new(Rows))
758///     .sampler(FnSampler::new(Some(1), |_| [0].into_iter()))
759///     .collate(VecCollate)
760///     .dataset_identity("rows".to_owned())
761///     .build();
762/// ```
763///
764/// ```compile_fail
765/// use std::convert::Infallible;
766/// use rusttorch_data::{
767///     DataLoader, Dataset, FnBatchSource, ReplaySafeDataset, ReplaySafeMap,
768///     VecCollate,
769/// };
770/// struct Rows;
771/// impl Dataset for Rows {
772///     type Sample = i64;
773///     type Error = Infallible;
774///     fn len(&self) -> usize { 1 }
775///     fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
776/// }
777/// impl ReplaySafeDataset for Rows {}
778/// let _ = DataLoader::builder(ReplaySafeMap::new(Rows))
779///     .batch_sampler(FnBatchSource::new(Some(1), |_| vec![vec![0]].into_iter()))
780///     .collate(VecCollate)
781///     .dataset_identity("rows".to_owned())
782///     .build();
783/// ```
784///
785/// Worker replay supports explicit immutable datasets and checkpointable
786/// transforms. Zero workers use the same tagged envelope:
787///
788/// ```
789/// use std::convert::Infallible;
790/// use rusttorch_data::{
791///     DataLoader, Dataset, ReplaySafeDataset, ReplaySafeMap, VecCollate,
792/// };
793/// struct Rows;
794/// impl Dataset for Rows {
795///     type Sample = i64;
796///     type Error = Infallible;
797///     fn len(&self) -> usize { 1 }
798///     fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
799/// }
800/// impl ReplaySafeDataset for Rows {}
801/// let mut loader = DataLoader::builder(ReplaySafeMap::new(Rows))
802///     .workers(0)
803///     .collate(VecCollate)
804///     .dataset_identity("rows".to_owned())
805///     .build().unwrap();
806/// let state = loader.iter().checkpoint().unwrap();
807/// let mut resumed = DataLoader::builder(ReplaySafeMap::new(Rows))
808///     .workers(0).collate(VecCollate).dataset_identity("rows".to_owned())
809///     .resume_from(state).build().unwrap();
810/// assert_eq!(resumed.iter().next().unwrap().unwrap(), vec![1]);
811/// ```
812#[derive(Clone, Debug)]
813pub struct CheckpointFresh {
814    pub(crate) identity: String,
815}
816
817/// Builder state carrying a typed checkpoint selected for resume.
818#[derive(Clone, Debug)]
819pub struct CheckpointResume<S> {
820    pub(crate) identity: String,
821    pub(crate) state: S,
822}
823
824/// Retained first serial cursor and transform for a resumed loader.
825#[doc(hidden)]
826pub struct PendingSerial<I, T> {
827    pub(crate) batches: I,
828    pub(crate) transform: T,
829    pub(crate) next_batch: u64,
830    pub(crate) next_logical_sample: u64,
831}
832
833/// Built-loader state enabling exact serial checkpoints.
834#[doc(hidden)]
835pub struct CheckpointActive<I, T> {
836    pub(crate) identity: String,
837    pub(crate) configuration: LoaderConfiguration,
838    pub(crate) pending: Option<PendingSerial<I, T>>,
839}
840
841/// Active-iterator state at an exact serial checkpoint boundary.
842#[doc(hidden)]
843pub struct CheckpointIteration {
844    pub(crate) identity: String,
845    pub(crate) configuration: LoaderConfiguration,
846    pub(crate) boundary_valid: bool,
847}
848
849/// One deterministic worker lane's committed transform state.
850#[derive(Clone, Debug, Deserialize, Serialize)]
851pub struct WorkerLaneState<S> {
852    /// Zero-based lane identity.
853    pub id: usize,
854    /// Transform state before this lane's next unconsumed task.
855    pub state: S,
856}
857
858/// Explicit zero-worker or positive-worker transform state.
859#[derive(Clone, Debug, Deserialize, Serialize)]
860pub enum WorkerTransformLanes<S> {
861    /// One transform running on the calling thread.
862    Serial(S),
863    /// Transform states in deterministic lane order.
864    Workers(Vec<WorkerLaneState<S>>),
865}
866
867/// Versioned worker replay envelope within schema-v1 loader state.
868///
869/// An opaque transform cannot enable worker checkpointing without an explicit
870/// adapter, even when its closure appears deterministic:
871///
872/// ```compile_fail
873/// use std::convert::Infallible;
874/// use rusttorch_data::{DataLoader, Dataset, FnTransform, ReplaySafeDataset, ReplaySafeMap, TaskContext, VecCollate};
875/// struct Rows;
876/// impl Dataset for Rows {
877///     type Sample = i64;
878///     type Error = Infallible;
879///     fn len(&self) -> usize { 1 }
880///     fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
881/// }
882/// impl ReplaySafeDataset for Rows {}
883/// let _ = DataLoader::builder(ReplaySafeMap::new(Rows))
884///     .workers(2).collate(VecCollate)
885///     .transform(FnTransform::new(|x: i64, _: &TaskContext| Ok::<_, Infallible>(x)))
886///     .dataset_identity("rows".to_owned()).build();
887/// ```
888#[derive(Clone, Debug, Deserialize, Serialize)]
889pub struct WorkerTransformState<S> {
890    /// Worker envelope version, currently one.
891    pub schema_version: u32,
892    /// Exact worker count.
893    pub workers: usize,
894    /// Effective prefetch factor; absent for zero workers.
895    pub prefetch_factor: Option<usize>,
896    /// Original transform factory initialization identity.
897    pub factory_generation: u64,
898    /// Current transport generation used to reject stale messages.
899    pub run_generation: u64,
900    /// Explicit lane identities and states.
901    pub lanes: WorkerTransformLanes<S>,
902}
903
904/// Retained worker checkpoint components for the next exact iteration.
905#[doc(hidden)]
906pub struct WorkerCheckpointActive<I, T, S> {
907    pub(crate) identity: String,
908    pub(crate) configuration: LoaderConfiguration,
909    pub(crate) batches: Option<I>,
910    pub(crate) serial: Option<T>,
911    pub(crate) barrier: Option<crate::worker_checkpoint::Barrier<S>>,
912    pub(crate) next_batch: u64,
913    pub(crate) next_logical_sample: u64,
914    pub(crate) factory_generation: u64,
915    pub(crate) run_generation: u64,
916}
917
918/// Committed coordinator and worker boundary for one exact iteration.
919#[doc(hidden)]
920pub struct WorkerCheckpointIteration<S, E> {
921    pub(crate) identity: String,
922    pub(crate) configuration: LoaderConfiguration,
923    pub(crate) barrier: Option<crate::worker_checkpoint::Barrier<S>>,
924    pub(crate) logical_ends: std::collections::BTreeMap<u64, u64>,
925    pub(crate) errors: std::collections::BTreeMap<u64, E>,
926    pub(crate) next_logical_sample: u64,
927    pub(crate) factory_generation: u64,
928    pub(crate) boundary_valid: bool,
929}
930
931impl Checkpointable for () {
932    type State = ();
933
934    fn save_state(&self) -> Self::State {}
935
936    fn validate_state(&self, _state: &Self::State) -> Result<()> {
937        Ok(())
938    }
939
940    fn load_validated(&mut self, _state: &Self::State) {}
941}