rusttorch_data/checkpoint.rs
1use std::{error::Error, fmt};
2
3use rusttorch_core::{Result, RustTorchError};
4use serde::{Deserialize, Serialize, de::DeserializeOwned};
5
6use crate::{TaskContext, Transform, TransformFactory, WorkerContext};
7
8/// Replay-safe source with self-contained state captured before each read.
9///
10/// Validation is read-only; applying accepted state must not fail or panic.
11/// Records must have strictly increasing per-shard global sequence IDs.
12pub trait CheckpointableSource:
13 Iterator<Item = std::result::Result<crate::WorkerRecord<Self::Sample>, Self::Error>>
14{
15 /// Owned decoded sample.
16 type Sample;
17 /// Owned state, independent of future decoder progress.
18 type State: Clone + Serialize + DeserializeOwned;
19 /// Preserved source error.
20 type Error;
21 /// Returns the failed read's stable global position without changing state.
22 ///
23 /// Called after `next()` returns an error. This position must obey the
24 /// shard's strictly increasing sequence contract and repeat after rollback.
25 fn error_sequence(&self, error: &Self::Error) -> crate::SequenceId;
26 /// Captures the boundary before the next read attempt.
27 fn snapshot(&self) -> Self::State;
28 /// Validates without changing the source.
29 fn validate_snapshot(&self, state: &Self::State) -> std::result::Result<(), Self::Error>;
30 /// Applies state previously accepted by validation.
31 fn restore_validated(&mut self, state: &Self::State);
32 /// Replaces the run cancellation/deadline context before resumed reads.
33 ///
34 /// Sources retaining a context must override this method and replace it.
35 /// Logical worker generation and seed remain unchanged. This must not fail.
36 fn set_run_context(&mut self, _context: WorkerContext) {}
37}
38
39/// Stable factory identity for explicitly checkpointable stream sources.
40///
41/// An ordinary factory cannot acquire exact replay just by setting an identity:
42///
43/// ```compile_fail
44/// use rusttorch_data::*;
45/// use std::convert::Infallible;
46/// struct Factory;
47/// impl WorkerSourceFactory for Factory {
48/// type Sample = usize;
49/// type Error = Infallible;
50/// type Source = std::iter::Empty<Result<WorkerRecord<usize>, Infallible>>;
51/// fn create(&self, _: WorkerContext) -> Result<Self::Source, Infallible> { Ok(std::iter::empty()) }
52/// }
53/// StreamDataLoaderBuilder::new(Factory).collate(VecCollate)
54/// .checkpointable("ordinary").build().unwrap();
55/// ```
56///
57/// Opaque transforms also need an explicit `WorkerCheckpoint` implementation or
58/// an explicit `StatelessWorker`/`TransactionalWorker` adapter:
59///
60/// ```compile_fail
61/// use rusttorch_data::*;
62/// use std::convert::Infallible;
63/// struct Source;
64/// impl Iterator for Source {
65/// type Item = Result<WorkerRecord<usize>, Infallible>;
66/// fn next(&mut self) -> Option<Self::Item> { None }
67/// }
68/// impl CheckpointableSource for Source {
69/// type Sample = usize;
70/// type Error = Infallible;
71/// type State = ();
72/// fn error_sequence(&self, error: &Infallible) -> SequenceId { match *error {} }
73/// fn snapshot(&self) {}
74/// fn validate_snapshot(&self, _: &()) -> Result<(), Infallible> { Ok(()) }
75/// fn restore_validated(&mut self, _: &()) {}
76/// }
77/// struct Factory;
78/// impl WorkerSourceFactory for Factory {
79/// type Sample = usize;
80/// type Error = Infallible;
81/// type Source = Source;
82/// fn create(&self, _: WorkerContext) -> Result<Source, Infallible> { Ok(Source) }
83/// }
84/// impl CheckpointSourceFactory for Factory { const CHECKPOINT_KIND: &'static str = "empty.v1"; }
85/// let transform = FnTransform::new(|sample: usize, _: &TaskContext| Ok::<_, Infallible>(sample));
86/// StreamDataLoaderBuilder::new(Factory).transform(transform).collate(VecCollate)
87/// .checkpointable("opaque").build().unwrap();
88/// ```
89pub trait CheckpointSourceFactory: crate::WorkerSourceFactory {
90 /// Stable source kind and state-format version, for example `records.v1`.
91 const CHECKPOINT_KIND: &'static str;
92}
93
94/// Exact streaming configuration, with no sampler state.
95#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
96pub struct StreamCheckpointConfiguration {
97 /// Positive shard count.
98 pub workers: usize,
99 /// Maximum unpublished records per shard.
100 pub prefetch_factor: usize,
101 /// Coordinator batch size.
102 pub batch_size: usize,
103 /// Global short-tail policy.
104 pub drop_last: bool,
105 /// Task randomness seed.
106 pub loader_seed: u64,
107 /// Distributed task rank.
108 pub rank: usize,
109 /// Requested pinning behavior.
110 pub pin_request: CheckpointPinRequest,
111 /// Effective pinning behavior.
112 pub pin_status: CheckpointPinStatus,
113}
114
115/// Paired source and transform boundary for one worker shard.
116#[derive(Clone, Debug, Deserialize, Serialize)]
117pub struct StreamLaneState<S, T> {
118 /// Unique worker index in canonical order.
119 pub id: usize,
120 /// Source state preceding its first unconsumed attempt.
121 pub source: S,
122 /// Transform state preceding the same attempt.
123 pub transform: T,
124}
125
126/// Versioned exact stream state at the next consumer-visible batch.
127#[derive(Clone, Debug, Deserialize, Serialize)]
128pub struct StreamLoaderState<S, T = (), C = ()> {
129 /// Stream schema version; currently one.
130 pub schema_version: u32,
131 /// Caller identity for exact source contents.
132 pub source_identity: String,
133 /// Stable factory kind/version.
134 pub factory_kind: String,
135 /// Task epoch.
136 pub epoch: u64,
137 /// Logical source and transform initialization generation.
138 pub source_generation: u64,
139 /// Transport generation, advanced by each checkpoint barrier.
140 pub transport_generation: u64,
141 /// Next visible batch index.
142 pub next_batch: u64,
143 /// Next expected global sequence ID.
144 pub next_sequence: u64,
145 /// Whether the last visible batch was the final incomplete global tail.
146 pub short_tail: bool,
147 /// Exact global source length when supplied by the factory.
148 pub source_length: Option<usize>,
149 /// Task RNG derivation version.
150 pub rng_derivation_version: u32,
151 /// Worker seed derivation version.
152 pub worker_seed_derivation_version: u32,
153 /// Exact runtime settings.
154 pub configuration: StreamCheckpointConfiguration,
155 /// Canonically ordered worker states; never contains decoded samples.
156 pub lanes: Vec<StreamLaneState<S, T>>,
157 /// Coordinator collation state.
158 pub coordinator: C,
159}
160
161/// Typed stream construction or transactional resume failure.
162#[derive(Debug, thiserror::Error)]
163pub enum StreamCheckpointBuildError<S, F> {
164 /// Static configuration rejection.
165 #[error(transparent)]
166 Configuration(#[from] RustTorchError),
167 /// Source creation or read-only state validation failed.
168 #[error("stream source on worker {worker} failed: {source}")]
169 Source {
170 /// Worker lane.
171 worker: usize,
172 /// Original source error.
173 #[source]
174 source: S,
175 },
176 /// Transform creation failed.
177 #[error("stream transform initialization on worker {worker} failed: {source}")]
178 TransformFactory {
179 /// Worker lane.
180 worker: usize,
181 /// Original factory error.
182 #[source]
183 source: F,
184 },
185 /// Transform state validation failed.
186 #[error("stream transform validation on worker {worker} failed: {source}")]
187 TransformState {
188 /// Worker lane.
189 worker: usize,
190 /// Original validation error.
191 #[source]
192 source: RustTorchError,
193 },
194 /// Initialization or checkpoint callback panicked.
195 #[error("stream checkpoint worker {worker} panicked")]
196 WorkerPanic {
197 /// Worker lane.
198 worker: usize,
199 },
200}
201
202/// Schema version emitted by serial map-loader checkpoints.
203pub const LOADER_STATE_SCHEMA_VERSION: u32 = 1;
204
205/// Pinning request recorded in a loader checkpoint.
206#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
207pub enum CheckpointPinRequest {
208 /// Pinning was not requested.
209 Disabled,
210 /// Pinning was requested only when a CUDA accelerator is available.
211 Auto,
212 /// Pinning was requested for one CUDA device.
213 ExplicitCuda {
214 /// CUDA device index.
215 index: usize,
216 },
217}
218
219/// Effective pinning behavior recorded in a loader checkpoint.
220#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
221pub enum CheckpointPinStatus {
222 /// Pinning was not requested.
223 Disabled,
224 /// Automatic pinning found no supported accelerator.
225 DisabledNoAccelerator,
226 /// Batches are pinned for one CUDA device.
227 Cuda {
228 /// CUDA device index.
229 index: usize,
230 },
231}
232
233/// Loader settings that must match when a serial checkpoint is resumed.
234#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
235pub struct LoaderConfiguration {
236 /// Automatic batch size, or `None` for explicit/no batching.
237 pub batch_size: Option<usize>,
238 /// Whether an incomplete automatic batch is omitted.
239 pub drop_last: bool,
240 /// Exact worker count; zero for serial execution.
241 pub workers: usize,
242 /// Effective worker prefetch factor, or `None` for serial execution.
243 pub prefetch_factor: Option<usize>,
244 /// Whether results follow sampler order.
245 pub in_order: bool,
246 /// Loader seed used by task-local random transforms.
247 pub loader_seed: u64,
248 /// Distributed task rank.
249 pub rank: usize,
250 /// Distributed replica count, or one for an unsharded sampler.
251 pub replicas: usize,
252 /// Distributed shuffle policy when applicable.
253 pub distributed_shuffle: Option<bool>,
254 /// Distributed sampler seed when applicable.
255 pub distributed_seed: Option<u64>,
256 /// Distributed tail policy when applicable.
257 pub distributed_drop_last: Option<bool>,
258 /// Stable RustTorch sampler/plan kind.
259 pub sampler_kind: String,
260 /// Requested pinning policy.
261 pub pin_request: CheckpointPinRequest,
262 /// Effective pinning status visible to consumers.
263 pub pin_status: CheckpointPinStatus,
264}
265
266/// Versioned state for the next batch visible from a map loader.
267#[derive(Clone, Debug, Deserialize, Serialize)]
268pub struct LoaderState<D, S, T = (), C = ()> {
269 /// State schema version.
270 pub schema_version: u32,
271 /// Caller-defined identity for the exact dataset contents.
272 pub dataset_identity: String,
273 /// Active sampler epoch.
274 pub epoch: u64,
275 /// Transport generation. Serial state is always zero.
276 pub iterator_generation: u64,
277 /// Next consumer-visible batch number.
278 pub next_batch: u64,
279 /// Number of logical sample occurrences already consumed.
280 pub next_logical_sample: u64,
281 /// Dataset component state.
282 pub dataset: D,
283 /// Active sampler or batch-source cursor state.
284 pub sampler: S,
285 /// Serial transform state or a versioned worker transform envelope.
286 pub transform: T,
287 /// Effective coordinator collator/converter state.
288 pub collate: C,
289 /// Task-RNG derivation version.
290 pub rng_derivation_version: u32,
291 /// Worker-seed derivation version.
292 pub worker_seed_derivation_version: u32,
293 /// Exact loader configuration identity.
294 pub configuration: LoaderConfiguration,
295}
296
297/// A component whose state can be validated before it is restored.
298///
299/// Validation must be read-only. `load_validated` must not fail or panic after
300/// the same component accepted the supplied state; violating either rule
301/// breaks the resume transaction contract.
302pub trait Checkpointable {
303 /// Owned serializable component state.
304 type State: Clone + Serialize + DeserializeOwned;
305
306 /// Captures the current component state.
307 fn save_state(&self) -> Self::State;
308
309 /// Checks state compatibility without mutating the component.
310 fn validate_state(&self, state: &Self::State) -> Result<()>;
311
312 /// Restores state previously accepted by [`Self::validate_state`].
313 fn load_validated(&mut self, state: &Self::State);
314}
315
316/// Marker for a map dataset whose fixed-identity fetches are replay-safe.
317pub trait ReplaySafeDataset: crate::Dataset {}
318
319/// Dataset state used by exact serial loader checkpoints.
320///
321/// Validation must be read-only, and restore must be infallible after matching
322/// validation succeeds.
323pub trait DatasetCheckpoint: crate::Dataset {
324 /// Owned serializable dataset state.
325 type State: Clone + Serialize + DeserializeOwned;
326
327 /// Captures dataset state at the visible boundary.
328 fn snapshot_dataset(&self) -> Self::State;
329
330 /// Checks state compatibility without mutation.
331 fn validate_dataset_state(&self, state: &Self::State) -> Result<()>;
332
333 /// Restores state accepted by [`Self::validate_dataset_state`].
334 fn restore_dataset_validated(&mut self, state: &Self::State);
335}
336
337/// Marker for a component with no mutable checkpoint state.
338pub trait Stateless {}
339
340/// Transactional state contract for a serial component.
341///
342/// Validation must be read-only, and restore must be infallible after matching
343/// validation succeeds.
344pub trait TransactionalCheckpoint {
345 /// Owned serializable component state.
346 type State: Clone + Serialize + DeserializeOwned;
347
348 /// Captures current component state.
349 fn snapshot(&self) -> Self::State;
350
351 /// Checks state compatibility without mutation.
352 fn validate_snapshot(&self, state: &Self::State) -> Result<()>;
353
354 /// Restores state accepted by [`Self::validate_snapshot`].
355 fn restore_validated(&mut self, state: &Self::State);
356}
357
358/// Checkpoint contract for one transform instance.
359///
360/// Serial checkpoints capture one instance; worker barriers restore each
361/// deterministic lane to its first unconsumed task.
362pub trait WorkerCheckpoint {
363 /// Owned serializable transform state.
364 type State: Clone + Serialize + DeserializeOwned;
365
366 /// Captures current transform state.
367 fn snapshot(&self) -> Self::State;
368
369 /// Checks state compatibility without mutation.
370 fn validate_snapshot(&self, state: &Self::State) -> Result<()>;
371
372 /// Restores state accepted by [`Self::validate_snapshot`].
373 fn restore_validated(&mut self, state: &Self::State);
374}
375
376/// Explicit unit-state adapter for a transform the caller asserts is stateless.
377///
378/// Construction is the opt-in assertion: mutable behavior that affects future
379/// outputs violates this adapter's exact-resume contract. This makes opaque
380/// closure transforms usable without requiring downstream orphan impls.
381#[derive(Clone, Copy, Debug, Default)]
382pub struct StatelessWorker<T> {
383 inner: T,
384}
385
386impl<T> StatelessWorker<T> {
387 /// Explicitly asserts that a transform has no state affecting replay.
388 pub fn new(inner: T) -> Self {
389 Self { inner }
390 }
391
392 /// Returns the wrapped transform.
393 pub fn into_inner(self) -> T {
394 self.inner
395 }
396}
397
398impl<Input, T> Transform<Input> for StatelessWorker<T>
399where
400 T: Transform<Input>,
401{
402 type Output = T::Output;
403 type Error = T::Error;
404
405 fn transform(
406 &mut self,
407 input: Input,
408 context: &TaskContext,
409 ) -> std::result::Result<Self::Output, Self::Error> {
410 self.inner.transform(input, context)
411 }
412}
413
414impl<T> WorkerCheckpoint for StatelessWorker<T> {
415 type State = ();
416
417 fn snapshot(&self) -> Self::State {}
418
419 fn validate_snapshot(&self, _state: &Self::State) -> Result<()> {
420 Ok(())
421 }
422
423 fn restore_validated(&mut self, _state: &Self::State) {}
424}
425
426/// Explicit transactional adapter for a stateful transform.
427#[derive(Clone, Copy, Debug, Default)]
428pub struct TransactionalWorker<T> {
429 inner: T,
430}
431
432impl<T> TransactionalWorker<T> {
433 /// Wraps a transform with transactional checkpoint support.
434 pub fn new(inner: T) -> Self {
435 Self { inner }
436 }
437
438 /// Returns the wrapped transform.
439 pub fn into_inner(self) -> T {
440 self.inner
441 }
442}
443
444impl<Input, T> Transform<Input> for TransactionalWorker<T>
445where
446 T: Transform<Input> + TransactionalCheckpoint,
447{
448 type Output = T::Output;
449 type Error = T::Error;
450
451 fn transform(
452 &mut self,
453 input: Input,
454 context: &TaskContext,
455 ) -> std::result::Result<Self::Output, Self::Error> {
456 self.inner.transform(input, context)
457 }
458}
459
460impl<T> WorkerCheckpoint for TransactionalWorker<T>
461where
462 T: TransactionalCheckpoint,
463{
464 type State = T::State;
465
466 fn snapshot(&self) -> Self::State {
467 self.inner.snapshot()
468 }
469
470 fn validate_snapshot(&self, state: &Self::State) -> Result<()> {
471 self.inner.validate_snapshot(state)
472 }
473
474 fn restore_validated(&mut self, state: &Self::State) {
475 self.inner.restore_validated(state);
476 }
477}
478
479#[doc(hidden)]
480#[derive(Clone, Copy, Debug)]
481pub struct StatelessTransformFactory<F>(pub(crate) F);
482
483impl<Input, F> TransformFactory<Input> for StatelessTransformFactory<F>
484where
485 F: TransformFactory<Input>,
486{
487 type Transform = StatelessWorker<F::Transform>;
488 type Error = F::Error;
489
490 fn create(
491 &self,
492 worker: Option<&WorkerContext>,
493 ) -> std::result::Result<Self::Transform, Self::Error> {
494 self.0.create(worker).map(StatelessWorker::new)
495 }
496}
497
498#[doc(hidden)]
499#[derive(Clone, Copy, Debug)]
500pub struct TransactionalTransformFactory<F>(pub(crate) F);
501
502impl<Input, F> TransformFactory<Input> for TransactionalTransformFactory<F>
503where
504 F: TransformFactory<Input>,
505 F::Transform: TransactionalCheckpoint,
506{
507 type Transform = TransactionalWorker<F::Transform>;
508 type Error = F::Error;
509
510 fn create(
511 &self,
512 worker: Option<&WorkerContext>,
513 ) -> std::result::Result<Self::Transform, Self::Error> {
514 self.0.create(worker).map(TransactionalWorker::new)
515 }
516}
517
518/// Resume-build error preserving either a RustTorch validation error or the
519/// transform factory's original error value.
520#[derive(Debug)]
521pub enum CheckpointBuildError<F> {
522 /// Loader configuration or component validation failed.
523 Configuration(RustTorchError),
524 /// Constructing the one retained serial transform failed.
525 TransformFactory(F),
526}
527
528impl<F> From<RustTorchError> for CheckpointBuildError<F> {
529 fn from(error: RustTorchError) -> Self {
530 Self::Configuration(error)
531 }
532}
533
534impl<F> fmt::Display for CheckpointBuildError<F>
535where
536 F: fmt::Display,
537{
538 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
539 match self {
540 Self::Configuration(error) => error.fmt(formatter),
541 Self::TransformFactory(error) => {
542 write!(
543 formatter,
544 "transform initialization failed while resuming: {error}"
545 )
546 }
547 }
548 }
549}
550
551impl<F> Error for CheckpointBuildError<F>
552where
553 F: Error + 'static,
554{
555 fn source(&self) -> Option<&(dyn Error + 'static)> {
556 match self {
557 Self::Configuration(error) => Some(error),
558 Self::TransformFactory(error) => Some(error),
559 }
560 }
561}
562
563/// Default builder/loader state with exact checkpointing disabled.
564#[derive(Clone, Copy, Debug, Default)]
565pub struct CheckpointDisabled;
566
567/// Builder state after a dataset identity enables a fresh exact checkpoint.
568///
569/// Exact checkpointing requires explicit replay evidence; an arbitrary
570/// dataset is not enabled merely by assigning an identity:
571///
572/// ```compile_fail
573/// use std::convert::Infallible;
574/// use rusttorch_data::{DataLoader, Dataset, VecCollate};
575/// struct Rows;
576/// impl Dataset for Rows {
577/// type Sample = i64;
578/// type Error = Infallible;
579/// fn len(&self) -> usize { 1 }
580/// fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
581/// }
582/// let _ = DataLoader::builder(Rows)
583/// .collate(VecCollate)
584/// .dataset_identity("rows".to_owned())
585/// .build();
586/// ```
587///
588/// Ordinary [`crate::TensorDataset`] rows share storage and are therefore not
589/// exact-checkpoint capable until [`crate::TensorDataset::into_replay_safe`]
590/// succeeds:
591///
592/// ```compile_fail
593/// use rusttorch_core::Tensor;
594/// use rusttorch_data::{DataLoader, TensorDataset, VecCollate};
595/// let dataset = TensorDataset::new(vec![Tensor::from_slice(&[1_i64])]).unwrap();
596/// let _ = DataLoader::builder(dataset)
597/// .collate(VecCollate)
598/// .dataset_identity("tensor".to_owned())
599/// .build();
600/// ```
601///
602/// An opaque closure transform has no restorable state contract unless the
603/// caller explicitly selects a stateless or transactional adapter:
604///
605/// ```compile_fail
606/// use std::convert::Infallible;
607/// use rusttorch_data::{
608/// DataLoader, Dataset, FnTransform, ReplaySafeDataset, ReplaySafeMap,
609/// TaskContext, VecCollate,
610/// };
611/// struct Rows;
612/// impl Dataset for Rows {
613/// type Sample = i64;
614/// type Error = Infallible;
615/// fn len(&self) -> usize { 1 }
616/// fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
617/// }
618/// impl ReplaySafeDataset for Rows {}
619/// let transform = FnTransform::new(|value, _: &TaskContext| Ok::<_, Infallible>(value));
620/// let _ = DataLoader::builder(ReplaySafeMap::new(Rows))
621/// .transform(transform)
622/// .collate(VecCollate)
623/// .dataset_identity("rows".to_owned())
624/// .build();
625/// ```
626///
627/// Closure-backed collation is likewise opaque without an explicit
628/// [`Checkpointable`] implementation:
629///
630/// ```compile_fail
631/// use std::convert::Infallible;
632/// use rusttorch_data::{
633/// DataLoader, Dataset, FnCollate, ReplaySafeDataset, ReplaySafeMap,
634/// };
635/// struct Rows;
636/// impl Dataset for Rows {
637/// type Sample = i64;
638/// type Error = Infallible;
639/// fn len(&self) -> usize { 1 }
640/// fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
641/// }
642/// impl ReplaySafeDataset for Rows {}
643/// let _ = DataLoader::builder(ReplaySafeMap::new(Rows))
644/// .collate(FnCollate::new(|values| Ok::<_, Infallible>(values)))
645/// .dataset_identity("rows".to_owned())
646/// .build();
647/// ```
648///
649/// Closure-backed sampler and explicit-batch sources do not expose stable
650/// configuration/cursor state:
651///
652/// ```compile_fail
653/// use std::convert::Infallible;
654/// use rusttorch_data::{
655/// DataLoader, Dataset, FnSampler, ReplaySafeDataset, ReplaySafeMap,
656/// VecCollate,
657/// };
658/// struct Rows;
659/// impl Dataset for Rows {
660/// type Sample = i64;
661/// type Error = Infallible;
662/// fn len(&self) -> usize { 1 }
663/// fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
664/// }
665/// impl ReplaySafeDataset for Rows {}
666/// let _ = DataLoader::builder(ReplaySafeMap::new(Rows))
667/// .sampler(FnSampler::new(Some(1), |_| [0].into_iter()))
668/// .collate(VecCollate)
669/// .dataset_identity("rows".to_owned())
670/// .build();
671/// ```
672///
673/// ```compile_fail
674/// use std::convert::Infallible;
675/// use rusttorch_data::{
676/// DataLoader, Dataset, FnBatchSource, ReplaySafeDataset, ReplaySafeMap,
677/// VecCollate,
678/// };
679/// struct Rows;
680/// impl Dataset for Rows {
681/// type Sample = i64;
682/// type Error = Infallible;
683/// fn len(&self) -> usize { 1 }
684/// fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
685/// }
686/// impl ReplaySafeDataset for Rows {}
687/// let _ = DataLoader::builder(ReplaySafeMap::new(Rows))
688/// .batch_sampler(FnBatchSource::new(Some(1), |_| vec![vec![0]].into_iter()))
689/// .collate(VecCollate)
690/// .dataset_identity("rows".to_owned())
691/// .build();
692/// ```
693///
694/// Worker replay supports explicit immutable datasets and checkpointable
695/// transforms. Zero workers use the same tagged envelope:
696///
697/// ```
698/// use std::convert::Infallible;
699/// use rusttorch_data::{
700/// DataLoader, Dataset, ReplaySafeDataset, ReplaySafeMap, VecCollate,
701/// };
702/// struct Rows;
703/// impl Dataset for Rows {
704/// type Sample = i64;
705/// type Error = Infallible;
706/// fn len(&self) -> usize { 1 }
707/// fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
708/// }
709/// impl ReplaySafeDataset for Rows {}
710/// let mut loader = DataLoader::builder(ReplaySafeMap::new(Rows))
711/// .workers(0)
712/// .collate(VecCollate)
713/// .dataset_identity("rows".to_owned())
714/// .build().unwrap();
715/// let state = loader.iter().checkpoint().unwrap();
716/// let mut resumed = DataLoader::builder(ReplaySafeMap::new(Rows))
717/// .workers(0).collate(VecCollate).dataset_identity("rows".to_owned())
718/// .resume_from(state).build().unwrap();
719/// assert_eq!(resumed.iter().next().unwrap().unwrap(), vec![1]);
720/// ```
721#[derive(Clone, Debug)]
722pub struct CheckpointFresh {
723 pub(crate) identity: String,
724}
725
726/// Builder state carrying a typed checkpoint selected for resume.
727#[derive(Clone, Debug)]
728pub struct CheckpointResume<S> {
729 pub(crate) identity: String,
730 pub(crate) state: S,
731}
732
733/// Retained first serial cursor and transform for a resumed loader.
734#[doc(hidden)]
735pub struct PendingSerial<I, T> {
736 pub(crate) batches: I,
737 pub(crate) transform: T,
738 pub(crate) next_batch: u64,
739 pub(crate) next_logical_sample: u64,
740}
741
742/// Built-loader state enabling exact serial checkpoints.
743#[doc(hidden)]
744pub struct CheckpointActive<I, T> {
745 pub(crate) identity: String,
746 pub(crate) configuration: LoaderConfiguration,
747 pub(crate) pending: Option<PendingSerial<I, T>>,
748}
749
750/// Active-iterator state at an exact serial checkpoint boundary.
751#[doc(hidden)]
752pub struct CheckpointIteration {
753 pub(crate) identity: String,
754 pub(crate) configuration: LoaderConfiguration,
755 pub(crate) boundary_valid: bool,
756}
757
758/// One deterministic worker lane's committed transform state.
759#[derive(Clone, Debug, Deserialize, Serialize)]
760pub struct WorkerLaneState<S> {
761 /// Zero-based lane identity.
762 pub id: usize,
763 /// Transform state before this lane's next unconsumed task.
764 pub state: S,
765}
766
767/// Explicit zero-worker or positive-worker transform state.
768#[derive(Clone, Debug, Deserialize, Serialize)]
769pub enum WorkerTransformLanes<S> {
770 /// One transform running on the calling thread.
771 Serial(S),
772 /// Transform states in deterministic lane order.
773 Workers(Vec<WorkerLaneState<S>>),
774}
775
776/// Versioned worker replay envelope within schema-v1 loader state.
777///
778/// An opaque transform cannot enable worker checkpointing without an explicit
779/// adapter, even when its closure appears deterministic:
780///
781/// ```compile_fail
782/// use std::convert::Infallible;
783/// use rusttorch_data::{DataLoader, Dataset, FnTransform, ReplaySafeDataset, ReplaySafeMap, TaskContext, VecCollate};
784/// struct Rows;
785/// impl Dataset for Rows {
786/// type Sample = i64;
787/// type Error = Infallible;
788/// fn len(&self) -> usize { 1 }
789/// fn get(&self, _: usize) -> Result<i64, Infallible> { Ok(1) }
790/// }
791/// impl ReplaySafeDataset for Rows {}
792/// let _ = DataLoader::builder(ReplaySafeMap::new(Rows))
793/// .workers(2).collate(VecCollate)
794/// .transform(FnTransform::new(|x: i64, _: &TaskContext| Ok::<_, Infallible>(x)))
795/// .dataset_identity("rows".to_owned()).build();
796/// ```
797#[derive(Clone, Debug, Deserialize, Serialize)]
798pub struct WorkerTransformState<S> {
799 /// Worker envelope version, currently one.
800 pub schema_version: u32,
801 /// Exact worker count.
802 pub workers: usize,
803 /// Effective prefetch factor; absent for zero workers.
804 pub prefetch_factor: Option<usize>,
805 /// Original transform factory initialization identity.
806 pub factory_generation: u64,
807 /// Current transport generation used to reject stale messages.
808 pub run_generation: u64,
809 /// Explicit lane identities and states.
810 pub lanes: WorkerTransformLanes<S>,
811}
812
813/// Retained worker checkpoint components for the next exact iteration.
814#[doc(hidden)]
815pub struct WorkerCheckpointActive<I, T, S> {
816 pub(crate) identity: String,
817 pub(crate) configuration: LoaderConfiguration,
818 pub(crate) batches: Option<I>,
819 pub(crate) serial: Option<T>,
820 pub(crate) barrier: Option<crate::worker_checkpoint::Barrier<S>>,
821 pub(crate) next_batch: u64,
822 pub(crate) next_logical_sample: u64,
823 pub(crate) factory_generation: u64,
824 pub(crate) run_generation: u64,
825}
826
827/// Committed coordinator and worker boundary for one exact iteration.
828#[doc(hidden)]
829pub struct WorkerCheckpointIteration<S, E> {
830 pub(crate) identity: String,
831 pub(crate) configuration: LoaderConfiguration,
832 pub(crate) barrier: Option<crate::worker_checkpoint::Barrier<S>>,
833 pub(crate) logical_ends: std::collections::BTreeMap<u64, u64>,
834 pub(crate) errors: std::collections::BTreeMap<u64, E>,
835 pub(crate) next_logical_sample: u64,
836 pub(crate) factory_generation: u64,
837 pub(crate) boundary_valid: bool,
838}
839
840impl Checkpointable for () {
841 type State = ();
842
843 fn save_state(&self) -> Self::State {}
844
845 fn validate_state(&self, _state: &Self::State) -> Result<()> {
846 Ok(())
847 }
848
849 fn load_validated(&mut self, _state: &Self::State) {}
850}