Skip to main content

rusttorch_data/
loader.rs

1use std::{
2    collections::BTreeMap,
3    convert::Infallible,
4    marker::PhantomData,
5    num::NonZeroUsize,
6    panic::{AssertUnwindSafe, catch_unwind},
7    sync::Arc,
8    time::Duration,
9};
10
11use rusttorch_core::{Device, Result, RustTorchError, available_devices};
12
13#[path = "loader_checkpoint.rs"]
14mod checkpoint_workers;
15
16use crate::checkpoint::{StatelessTransformFactory, TransactionalTransformFactory};
17use crate::checkpoint::{
18    WorkerCheckpointActive, WorkerCheckpointIteration, WorkerLaneState, WorkerTransformLanes,
19    WorkerTransformState,
20};
21use crate::memory::{ByteBudget, MemoryDisabled, MemoryEnabled, MemoryPolicy};
22use crate::sampler::{
23    BatchSourceCheckpoint, DistributedConfiguration, SamplerCheckpoint, validate_batch_size,
24};
25use crate::worker::{
26    WorkerBatch, WorkerFailure, WorkerMessage, WorkerPool, WorkerPoolConfiguration, WorkerReceive,
27    WorkerRunContext, WorkerSubmit, WorkerTask, validate_worker_pool_capacity,
28};
29use crate::{
30    Auto, BatchSampler, BatchSource, CheckpointActive, CheckpointBuildError, CheckpointDisabled,
31    CheckpointFresh, CheckpointIteration, CheckpointPinRequest, CheckpointPinStatus,
32    CheckpointResume, Checkpointable, CloneTransformFactory, Collate, Dataset, DatasetCheckpoint,
33    Deadline, DefaultCollator, DefaultConverter, Explicit, IdentityTransformFactory,
34    LOADER_STATE_SCHEMA_VERSION, LoaderError, LoaderState, MemoryFootprint, NoWorkerInit,
35    PinDisabled, PinEnabled, PinMemory, PinMemoryStatus, PipelineError, RandomSampler, Sampler,
36    SequentialSampler, TASK_RNG_DERIVATION_VERSION, TaskContext, Transform, TransformFactory,
37    WORKER_SEED_DERIVATION_VERSION, WorkerCheckpoint, WorkerInit,
38};
39
40/// Type-level marker used only to make [`crate::DataLoader::builder`] inferable.
41#[doc(hidden)]
42pub struct BuilderDatasetMarker {
43    _private: (),
44}
45
46/// Default execution capability preserving fully local serial data types.
47#[derive(Clone, Copy, Debug, Default)]
48pub struct SerialExecution;
49
50/// Execution capability selected by [`DataLoaderBuilder::workers`].
51#[derive(Clone, Copy, Debug, Default)]
52pub struct WorkerExecution;
53
54trait SerialBoundaryPolicy {
55    fn begin_next(&mut self);
56    fn commit_visible(&mut self);
57}
58
59trait WorkerBoundaryPolicy<E> {
60    fn begin_next(&mut self) {}
61    fn submitted(&mut self, _sequence: u64, _logical_end: u64) {}
62    fn committed(&mut self, _sequence: u64, _serial_logical_end: u64) {}
63    fn defer_error(&mut self, _sequence: u64, error: E) -> Option<E> {
64        Some(error)
65    }
66    fn take_error(&mut self, _sequence: u64) -> Option<E> {
67        None
68    }
69}
70impl<E> WorkerBoundaryPolicy<E> for CheckpointDisabled {}
71impl<S, E> WorkerBoundaryPolicy<E> for WorkerCheckpointIteration<S, E> {
72    fn begin_next(&mut self) {
73        self.boundary_valid = false;
74    }
75    fn submitted(&mut self, sequence: u64, logical_end: u64) {
76        self.logical_ends.insert(sequence, logical_end);
77    }
78    fn committed(&mut self, sequence: u64, serial_logical_end: u64) {
79        self.next_logical_sample = self
80            .logical_ends
81            .remove(&sequence)
82            .unwrap_or(serial_logical_end);
83        self.boundary_valid = true;
84        if let Some(barrier) = &self.barrier {
85            barrier
86                .committed
87                .store(sequence + 1, std::sync::atomic::Ordering::Release);
88        }
89    }
90    fn defer_error(&mut self, sequence: u64, error: E) -> Option<E> {
91        self.errors.insert(sequence, error);
92        None
93    }
94    fn take_error(&mut self, sequence: u64) -> Option<E> {
95        self.errors.remove(&sequence)
96    }
97}
98
99impl SerialBoundaryPolicy for CheckpointDisabled {
100    fn begin_next(&mut self) {}
101    fn commit_visible(&mut self) {}
102}
103
104impl SerialBoundaryPolicy for CheckpointIteration {
105    fn begin_next(&mut self) {
106        self.boundary_valid = false;
107    }
108
109    fn commit_visible(&mut self) {
110        self.boundary_valid = true;
111    }
112}
113
114type TransformOutput<D, F> =
115    <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<
116        <D as Dataset>::Sample,
117    >>::Output;
118type TransformFailure<D, F> =
119    <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<
120        <D as Dataset>::Sample,
121    >>::Error;
122type PlanFailure<D, P, C, F> = <P as LoaderPlan<TransformOutput<D, F>, C>>::Error;
123type IterPipelineError<D, P, C, F, I> = PipelineError<
124    <D as Dataset>::Error,
125    TransformFailure<D, F>,
126    PlanFailure<D, P, C, F>,
127    <F as TransformFactory<<D as Dataset>::Sample>>::Error,
128    <I as WorkerInit>::Error,
129>;
130type IterError<D, P, C, F, I> = LoaderError<IterPipelineError<D, P, C, F, I>>;
131type IterResult<D, P, C, F, I> = std::result::Result<
132    <P as LoaderPlan<TransformOutput<D, F>, C>>::Batch,
133    IterError<D, P, C, F, I>,
134>;
135type CompletedBatch<D, F, M> = (
136    usize,
137    WorkerBatch<TransformOutput<D, F>, <M as MemoryPolicy>::Permit>,
138);
139type FootprintFn<D, F> = fn(&[TransformOutput<D, F>]) -> usize;
140type WorkerStageFailure<D, F, I> = WorkerFailure<
141    <D as Dataset>::Error,
142    TransformFailure<D, F>,
143    <F as TransformFactory<<D as Dataset>::Sample>>::Error,
144    <I as WorkerInit>::Error,
145>;
146
147#[doc(hidden)]
148pub trait MapPinPolicy<D, P, C, F>
149where
150    D: Dataset,
151    F: TransformFactory<D::Sample>,
152    P: LoaderPlan<TransformOutput<D, F>, C>,
153{
154    fn pin(batch: P::Batch, device: Device) -> Result<P::Batch>;
155}
156
157impl<D, P, C, F> MapPinPolicy<D, P, C, F> for PinDisabled
158where
159    D: Dataset,
160    F: TransformFactory<D::Sample>,
161    P: LoaderPlan<TransformOutput<D, F>, C>,
162{
163    fn pin(batch: P::Batch, _device: Device) -> Result<P::Batch> {
164        Ok(batch)
165    }
166}
167
168impl<D, P, C, F, Q> MapPinPolicy<D, P, C, F> for PinEnabled<Q>
169where
170    D: Dataset,
171    F: TransformFactory<D::Sample>,
172    P: LoaderPlan<TransformOutput<D, F>, C>,
173    P::Batch: PinMemory,
174{
175    fn pin(batch: P::Batch, device: Device) -> Result<P::Batch> {
176        batch.pin_memory(device)
177    }
178}
179
180struct PendingSubmission {
181    worker: usize,
182    task: WorkerTask,
183    next_submission: u64,
184    logical_end: u64,
185}
186
187impl Dataset for BuilderDatasetMarker {
188    type Sample = ();
189    type Error = Infallible;
190
191    fn len(&self) -> usize {
192        0
193    }
194
195    fn get(&self, _index: usize) -> std::result::Result<Self::Sample, Self::Error> {
196        unreachable!("the DataLoader builder marker never loads samples")
197    }
198}
199
200/// Automatic fixed-size batching over a reusable sampler.
201pub struct AutoBatch<S> {
202    sampler: S,
203    batch_size: NonZeroUsize,
204    drop_last: bool,
205}
206
207/// Explicit reusable batches of indices.
208pub struct ExplicitBatches<B> {
209    batches: B,
210}
211
212/// One-sample loading without an automatic batch dimension.
213pub struct NoBatch<S, V = DefaultConverter> {
214    sampler: S,
215    converter: V,
216}
217
218/// Internal plan behavior used by the named owned-loader iterator.
219#[doc(hidden)]
220pub trait LoaderPlan<Sample, C> {
221    /// Fresh iterator of index groups.
222    type Iter: Iterator<Item = Vec<usize>>;
223    /// Item yielded by the loader.
224    type Batch;
225    /// Typed dataset/collation pipeline error.
226    type Error;
227
228    /// Panic stage used while reading the plan epoch.
229    const EPOCH_PANIC_STAGE: &'static str = "sampler epoch";
230    /// Panic stage used while creating a fresh plan iterator.
231    const CREATION_PANIC_STAGE: &'static str = "sampler creation";
232    /// Panic stage used while refilling from the plan iterator.
233    const REFILL_PANIC_STAGE: &'static str = "sampler refill";
234    /// Panic stage used while producing the visible item.
235    const FINISH_PANIC_STAGE: &'static str = "collation";
236
237    /// Creates fresh index groups for the current epoch.
238    fn iter(&self) -> Self::Iter;
239    /// Returns the exact output count when known.
240    fn exact_len(&self) -> Option<usize>;
241    /// Returns the configured epoch.
242    fn epoch(&self) -> u64;
243    /// Forwards a new epoch to the reusable source.
244    fn set_epoch(&mut self, epoch: u64);
245    /// Converts fetched samples into one visible item.
246    fn finish(
247        &mut self,
248        collator: &mut C,
249        samples: Vec<Sample>,
250    ) -> std::result::Result<Self::Batch, Self::Error>;
251}
252
253/// Configuration-only behavior shared by loader index plans.
254#[doc(hidden)]
255pub trait LoaderPlanConfiguration {
256    /// Applies builder-level automatic batch controls when relevant.
257    fn apply_batch_options(&mut self, batch_size: NonZeroUsize, drop_last: bool);
258    /// Applies the initial epoch selected by the builder.
259    fn apply_epoch(&mut self, epoch: u64);
260}
261
262#[doc(hidden)]
263#[derive(Clone)]
264pub struct PlanCheckpointIdentity {
265    batch_size: Option<usize>,
266    drop_last: bool,
267    sampler_kind: String,
268    distributed: Option<DistributedConfiguration>,
269}
270
271#[doc(hidden)]
272pub trait CheckpointPlan<Sample, C>: LoaderPlan<Sample, C> {
273    type State;
274    type CoordinatorState;
275
276    const AUTOMATIC_BATCHING: bool = false;
277
278    fn checkpoint_identity(&self) -> PlanCheckpointIdentity;
279    /// Derives resume identity without observing a live plan or component.
280    fn checkpoint_identity_from_state(
281        state: &Self::State,
282        batch_size: NonZeroUsize,
283        drop_last: bool,
284    ) -> Result<PlanCheckpointIdentity>;
285    fn checkpoint_state(&self, next_batch: u64, next_logical_sample: u64) -> Result<Self::State>;
286    fn validate_checkpoint_state(
287        &self,
288        state: &Self::State,
289        epoch: u64,
290        next_batch: u64,
291        next_logical_sample: u64,
292    ) -> Result<()>;
293    fn validate_checkpoint_state_with_batch_options(
294        &self,
295        state: &Self::State,
296        epoch: u64,
297        next_batch: u64,
298        next_logical_sample: u64,
299        _batch_size: NonZeroUsize,
300        _drop_last: bool,
301    ) -> Result<()> {
302        self.validate_checkpoint_state(state, epoch, next_batch, next_logical_sample)
303    }
304    fn restore_checkpoint_iter_validated(&mut self, state: &Self::State) -> Self::Iter;
305    fn save_coordinator(&self, collator: &C) -> Self::CoordinatorState;
306    fn validate_coordinator(&self, collator: &C, state: &Self::CoordinatorState) -> Result<()>;
307    fn restore_coordinator_validated(&mut self, collator: &mut C, state: &Self::CoordinatorState);
308}
309
310impl<Sample, S, C> LoaderPlan<Sample, C> for AutoBatch<S>
311where
312    S: Sampler,
313    C: Collate<Sample>,
314{
315    type Iter = BatchSampler<S::Iter>;
316    type Batch = C::Batch;
317    type Error = C::Error;
318
319    const EPOCH_PANIC_STAGE: &'static str = "sampler epoch";
320    const CREATION_PANIC_STAGE: &'static str = "sampler creation";
321    const REFILL_PANIC_STAGE: &'static str = "sampler refill";
322    const FINISH_PANIC_STAGE: &'static str = "collation";
323
324    fn iter(&self) -> Self::Iter {
325        BatchSampler::from_validated(self.sampler.iter(), self.batch_size, self.drop_last)
326    }
327
328    fn exact_len(&self) -> Option<usize> {
329        self.sampler
330            .exact_len()
331            .map(|length| batch_count(length, self.batch_size, self.drop_last))
332    }
333
334    fn epoch(&self) -> u64 {
335        self.sampler.epoch()
336    }
337
338    fn set_epoch(&mut self, epoch: u64) {
339        self.sampler.set_epoch(epoch);
340    }
341
342    fn finish(
343        &mut self,
344        collator: &mut C,
345        samples: Vec<Sample>,
346    ) -> std::result::Result<Self::Batch, Self::Error> {
347        collator.collate(samples)
348    }
349}
350
351impl<S> LoaderPlanConfiguration for AutoBatch<S>
352where
353    S: Sampler,
354{
355    fn apply_batch_options(&mut self, batch_size: NonZeroUsize, drop_last: bool) {
356        self.batch_size = batch_size;
357        self.drop_last = drop_last;
358    }
359
360    fn apply_epoch(&mut self, epoch: u64) {
361        self.sampler.set_epoch(epoch);
362    }
363}
364
365impl<Sample, S, C> CheckpointPlan<Sample, C> for AutoBatch<S>
366where
367    S: SamplerCheckpoint,
368    C: Collate<Sample> + Checkpointable,
369{
370    type State = S::State;
371    type CoordinatorState = C::State;
372
373    const AUTOMATIC_BATCHING: bool = true;
374
375    fn checkpoint_identity(&self) -> PlanCheckpointIdentity {
376        PlanCheckpointIdentity {
377            batch_size: Some(self.batch_size.get()),
378            drop_last: self.drop_last,
379            sampler_kind: self.sampler.kind().to_owned(),
380            distributed: self.sampler.distributed_configuration(),
381        }
382    }
383
384    fn checkpoint_identity_from_state(
385        state: &Self::State,
386        batch_size: NonZeroUsize,
387        drop_last: bool,
388    ) -> Result<PlanCheckpointIdentity> {
389        Ok(PlanCheckpointIdentity {
390            batch_size: Some(batch_size.get()),
391            drop_last,
392            sampler_kind: S::checkpoint_kind_from_state(state),
393            distributed: S::checkpoint_distributed_configuration_from_state(state)?,
394        })
395    }
396
397    fn checkpoint_state(&self, next_batch: u64, next_logical_sample: u64) -> Result<Self::State> {
398        validate_auto_batch_boundary(
399            self.sampler.exact_len(),
400            self.batch_size,
401            self.drop_last,
402            next_batch,
403            next_logical_sample,
404        )?;
405        self.sampler.checkpoint_state(next_logical_sample)
406    }
407
408    fn validate_checkpoint_state(
409        &self,
410        state: &Self::State,
411        epoch: u64,
412        next_batch: u64,
413        next_logical_sample: u64,
414    ) -> Result<()> {
415        <Self as CheckpointPlan<Sample, C>>::validate_checkpoint_state_with_batch_options(
416            self,
417            state,
418            epoch,
419            next_batch,
420            next_logical_sample,
421            self.batch_size,
422            self.drop_last,
423        )
424    }
425
426    fn validate_checkpoint_state_with_batch_options(
427        &self,
428        state: &Self::State,
429        epoch: u64,
430        next_batch: u64,
431        next_logical_sample: u64,
432        batch_size: NonZeroUsize,
433        drop_last: bool,
434    ) -> Result<()> {
435        validate_auto_batch_boundary(
436            self.sampler.exact_len(),
437            batch_size,
438            drop_last,
439            next_batch,
440            next_logical_sample,
441        )?;
442        self.sampler
443            .validate_checkpoint_state(state, epoch, next_logical_sample)
444    }
445
446    fn restore_checkpoint_iter_validated(&mut self, state: &Self::State) -> Self::Iter {
447        let iterator = self.sampler.restore_checkpoint_iter_validated(state);
448        BatchSampler::from_validated(iterator, self.batch_size, self.drop_last)
449    }
450
451    fn save_coordinator(&self, collator: &C) -> Self::CoordinatorState {
452        collator.save_state()
453    }
454
455    fn validate_coordinator(&self, collator: &C, state: &Self::CoordinatorState) -> Result<()> {
456        collator.validate_state(state)
457    }
458
459    fn restore_coordinator_validated(&mut self, collator: &mut C, state: &Self::CoordinatorState) {
460        collator.load_validated(state);
461    }
462}
463
464impl<Sample, B, C> LoaderPlan<Sample, C> for ExplicitBatches<B>
465where
466    B: BatchSource,
467    C: Collate<Sample>,
468{
469    type Iter = B::Iter;
470    type Batch = C::Batch;
471    type Error = C::Error;
472
473    const EPOCH_PANIC_STAGE: &'static str = "batch source epoch";
474    const CREATION_PANIC_STAGE: &'static str = "batch source creation";
475    const REFILL_PANIC_STAGE: &'static str = "batch source refill";
476    const FINISH_PANIC_STAGE: &'static str = "collation";
477
478    fn iter(&self) -> Self::Iter {
479        self.batches.iter()
480    }
481
482    fn exact_len(&self) -> Option<usize> {
483        self.batches.exact_len()
484    }
485
486    fn epoch(&self) -> u64 {
487        self.batches.epoch()
488    }
489
490    fn set_epoch(&mut self, epoch: u64) {
491        self.batches.set_epoch(epoch);
492    }
493
494    fn finish(
495        &mut self,
496        collator: &mut C,
497        samples: Vec<Sample>,
498    ) -> std::result::Result<Self::Batch, Self::Error> {
499        collator.collate(samples)
500    }
501}
502
503impl<B> LoaderPlanConfiguration for ExplicitBatches<B>
504where
505    B: BatchSource,
506{
507    fn apply_batch_options(&mut self, _batch_size: NonZeroUsize, _drop_last: bool) {}
508
509    fn apply_epoch(&mut self, epoch: u64) {
510        self.batches.set_epoch(epoch);
511    }
512}
513
514impl<Sample, B, C> CheckpointPlan<Sample, C> for ExplicitBatches<B>
515where
516    B: BatchSourceCheckpoint,
517    C: Collate<Sample> + Checkpointable,
518{
519    type State = B::State;
520    type CoordinatorState = C::State;
521
522    fn checkpoint_identity(&self) -> PlanCheckpointIdentity {
523        PlanCheckpointIdentity {
524            batch_size: None,
525            drop_last: false,
526            sampler_kind: self.batches.kind(),
527            distributed: self.batches.distributed_configuration(),
528        }
529    }
530
531    fn checkpoint_identity_from_state(
532        state: &Self::State,
533        _batch_size: NonZeroUsize,
534        _drop_last: bool,
535    ) -> Result<PlanCheckpointIdentity> {
536        Ok(PlanCheckpointIdentity {
537            batch_size: None,
538            drop_last: false,
539            sampler_kind: B::checkpoint_kind_from_state(state),
540            distributed: B::checkpoint_distributed_configuration_from_state(state)?,
541        })
542    }
543
544    fn checkpoint_state(&self, next_batch: u64, next_logical_sample: u64) -> Result<Self::State> {
545        self.batches
546            .checkpoint_state(next_batch, next_logical_sample)
547    }
548
549    fn validate_checkpoint_state(
550        &self,
551        state: &Self::State,
552        epoch: u64,
553        next_batch: u64,
554        next_logical_sample: u64,
555    ) -> Result<()> {
556        self.batches
557            .validate_checkpoint_state(state, epoch, next_batch, next_logical_sample)
558    }
559
560    fn restore_checkpoint_iter_validated(&mut self, state: &Self::State) -> Self::Iter {
561        self.batches.restore_checkpoint_iter_validated(state)
562    }
563
564    fn save_coordinator(&self, collator: &C) -> Self::CoordinatorState {
565        collator.save_state()
566    }
567
568    fn validate_coordinator(&self, collator: &C, state: &Self::CoordinatorState) -> Result<()> {
569        collator.validate_state(state)
570    }
571
572    fn restore_coordinator_validated(&mut self, collator: &mut C, state: &Self::CoordinatorState) {
573        collator.load_validated(state);
574    }
575}
576
577fn singleton(index: usize) -> Vec<usize> {
578    vec![index]
579}
580
581impl<Sample, S, V, C> LoaderPlan<Sample, C> for NoBatch<S, V>
582where
583    S: Sampler,
584    V: Collate<Sample>,
585{
586    type Iter = std::iter::Map<S::Iter, fn(usize) -> Vec<usize>>;
587    type Batch = V::Batch;
588    type Error = V::Error;
589
590    const EPOCH_PANIC_STAGE: &'static str = "sampler epoch";
591    const CREATION_PANIC_STAGE: &'static str = "sampler creation";
592    const REFILL_PANIC_STAGE: &'static str = "sampler refill";
593    const FINISH_PANIC_STAGE: &'static str = "conversion";
594
595    fn iter(&self) -> Self::Iter {
596        self.sampler.iter().map(singleton)
597    }
598
599    fn exact_len(&self) -> Option<usize> {
600        self.sampler.exact_len()
601    }
602
603    fn epoch(&self) -> u64 {
604        self.sampler.epoch()
605    }
606
607    fn set_epoch(&mut self, epoch: u64) {
608        self.sampler.set_epoch(epoch);
609    }
610
611    fn finish(
612        &mut self,
613        _collator: &mut C,
614        samples: Vec<Sample>,
615    ) -> std::result::Result<Self::Batch, Self::Error> {
616        self.converter.collate(samples)
617    }
618}
619
620impl<S, V> LoaderPlanConfiguration for NoBatch<S, V>
621where
622    S: Sampler,
623{
624    fn apply_batch_options(&mut self, _batch_size: NonZeroUsize, _drop_last: bool) {}
625
626    fn apply_epoch(&mut self, epoch: u64) {
627        self.sampler.set_epoch(epoch);
628    }
629}
630
631impl<Sample, S, V, C> CheckpointPlan<Sample, C> for NoBatch<S, V>
632where
633    S: SamplerCheckpoint,
634    V: Collate<Sample> + Checkpointable,
635{
636    type State = S::State;
637    type CoordinatorState = V::State;
638
639    fn checkpoint_identity(&self) -> PlanCheckpointIdentity {
640        PlanCheckpointIdentity {
641            batch_size: None,
642            drop_last: false,
643            sampler_kind: self.sampler.kind().to_owned(),
644            distributed: self.sampler.distributed_configuration(),
645        }
646    }
647
648    fn checkpoint_identity_from_state(
649        state: &Self::State,
650        _batch_size: NonZeroUsize,
651        _drop_last: bool,
652    ) -> Result<PlanCheckpointIdentity> {
653        Ok(PlanCheckpointIdentity {
654            batch_size: None,
655            drop_last: false,
656            sampler_kind: S::checkpoint_kind_from_state(state),
657            distributed: S::checkpoint_distributed_configuration_from_state(state)?,
658        })
659    }
660
661    fn checkpoint_state(&self, next_batch: u64, next_logical_sample: u64) -> Result<Self::State> {
662        if next_batch != next_logical_sample {
663            return Err(invalid_configuration(
664                "checkpoint",
665                "no-batch cursor must advance one batch per logical sample",
666            ));
667        }
668        self.sampler.checkpoint_state(next_logical_sample)
669    }
670
671    fn validate_checkpoint_state(
672        &self,
673        state: &Self::State,
674        epoch: u64,
675        next_batch: u64,
676        next_logical_sample: u64,
677    ) -> Result<()> {
678        if next_batch != next_logical_sample {
679            return Err(invalid_configuration(
680                "checkpoint",
681                "no-batch cursor must advance one batch per logical sample",
682            ));
683        }
684        self.sampler
685            .validate_checkpoint_state(state, epoch, next_logical_sample)
686    }
687
688    fn restore_checkpoint_iter_validated(&mut self, state: &Self::State) -> Self::Iter {
689        self.sampler
690            .restore_checkpoint_iter_validated(state)
691            .map(singleton as fn(usize) -> Vec<usize>)
692    }
693
694    fn save_coordinator(&self, _collator: &C) -> Self::CoordinatorState {
695        self.converter.save_state()
696    }
697
698    fn validate_coordinator(&self, _collator: &C, state: &Self::CoordinatorState) -> Result<()> {
699        self.converter.validate_state(state)
700    }
701
702    fn restore_coordinator_validated(&mut self, _collator: &mut C, state: &Self::CoordinatorState) {
703        self.converter.load_validated(state);
704    }
705}
706
707#[derive(Clone, Copy, Default)]
708struct ExplicitArguments {
709    batch_size: bool,
710    drop_last: bool,
711    sampler: bool,
712    shuffle: bool,
713    batch_sampler: bool,
714    without_batching: bool,
715    prefetch_factor: bool,
716    prefetch_bytes: bool,
717}
718
719#[derive(Clone, Copy)]
720enum PinRequest {
721    Disabled,
722    Auto,
723    Explicit(Device),
724}
725
726#[derive(Clone, Copy)]
727struct BuilderConfiguration {
728    batch_size: usize,
729    drop_last: bool,
730    workers: usize,
731    persistent_workers: bool,
732    timeout: Option<Duration>,
733    ordered: bool,
734    pin_memory: PinRequest,
735    prefetch_factor: Option<usize>,
736    prefetch_bytes: Option<NonZeroUsize>,
737    loader_seed: u64,
738    epoch: u64,
739    rank: usize,
740}
741
742impl Default for BuilderConfiguration {
743    fn default() -> Self {
744        Self {
745            batch_size: 1,
746            drop_last: false,
747            workers: 0,
748            persistent_workers: false,
749            timeout: None,
750            ordered: true,
751            pin_memory: PinRequest::Disabled,
752            prefetch_factor: None,
753            prefetch_bytes: None,
754            loader_seed: 0,
755            epoch: 0,
756            rank: 0,
757        }
758    }
759}
760
761/// Configure repeated passes over an owned indexed dataset.
762///
763/// Create this with [`crate::DataLoader::builder`]. By default, samples arrive
764/// in index order, batch size is one, the final short batch is kept, and work
765/// runs on the calling thread. [`crate::DefaultCollator`] stacks tensors and
766/// numeric values recursively while preserving tuple fields.
767///
768/// # Shuffle each epoch and prepare batches with workers
769///
770/// Keep immutable metadata in the dataset and do decoding in `get()`. A positive
771/// worker count overlaps fetching and per-sample transforms; collation remains
772/// on the calling thread. The example uses small Rust records so its sampling
773/// and ownership behavior is visible without a decoder.
774///
775/// ```
776/// use std::convert::Infallible;
777/// use rusttorch_data::{DataLoader, Dataset, VecCollate};
778/// struct Rows;
779/// impl Dataset for Rows {
780///     type Sample = i64;
781///     type Error = Infallible;
782///     fn len(&self) -> usize { 6 }
783///     fn get(&self, index: usize) -> Result<i64, Infallible> { Ok(index as i64) }
784/// }
785/// fn main() -> Result<(), Box<dyn std::error::Error>> {
786///     let mut loader = DataLoader::builder(Rows)
787///         .batch_size(2).shuffle(42)?.workers(2).prefetch_factor(2)
788///         .collate(VecCollate).build()?;
789///     for epoch in 0..3 {
790///         loader.set_epoch(epoch);
791///         let batches = loader.iter().collect::<Result<Vec<_>, _>>()?;
792///         assert_eq!(batches.len(), 3);
793///         let mut visited: Vec<_> = batches.into_iter().flatten().collect();
794///         visited.sort_unstable();
795///         assert_eq!(visited, [0, 1, 2, 3, 4, 5]);
796///     }
797///     Ok(())
798/// }
799/// ```
800///
801/// Repeating an epoch repeats its sampling order. [`Self::shuffle`] sets the
802/// sampler seed; [`Self::seed`] separately seeds worker and sample-transform
803/// contexts. Omit `.workers(...)` for datasets that are not `Send + Sync`,
804/// including [`crate::TensorDataset`]. Calling `.workers(0)` still selects the
805/// worker-safe API and requires those bounds.
806///
807/// # Pad variable-length sequences in a custom batch
808///
809/// [`crate::FnCollate`] receives the selected samples after transforms. Return
810/// your model's batch representation, including lengths or padding masks when
811/// needed. Here zero is the padding value and lengths identify real tokens.
812///
813/// ```
814/// use std::convert::Infallible;
815/// use rusttorch_core::Tensor;
816/// use rusttorch_data::{DataLoader, Dataset, FnCollate};
817/// struct Sentences(Vec<Vec<i64>>);
818/// impl Dataset for Sentences {
819///     type Sample = Vec<i64>;
820///     type Error = Infallible;
821///     fn len(&self) -> usize { self.0.len() }
822///     fn get(&self, index: usize) -> Result<Self::Sample, Infallible> {
823///         Ok(self.0[index].clone())
824///     }
825/// }
826/// fn main() -> Result<(), Box<dyn std::error::Error>> {
827///     let padding = FnCollate::new(|rows: Vec<Vec<i64>>| {
828///         let width = rows.iter().map(Vec::len).max().unwrap_or(0);
829///         let lengths: Vec<_> = rows.iter().map(|row| row.len() as i64).collect();
830///         let count = rows.len();
831///         let mut values = Vec::new();
832///         for mut row in rows {
833///             row.resize(width, 0);
834///             values.extend(row);
835///         }
836///         Ok::<_, rusttorch_core::RustTorchError>((
837///             Tensor::f_from_slice(&values)?.f_reshape([count as i64, width as i64])?,
838///             Tensor::f_from_slice(&lengths)?,
839///         ))
840///     });
841///     let mut loader = DataLoader::builder(Sentences(vec![vec![1, 2], vec![3, 4, 5]]))
842///         .batch_size(2).collate(padding).build()?;
843///     let (tokens, lengths) = loader.iter().next().unwrap()?;
844///     assert_eq!(Vec::<Vec<i64>>::try_from(&tokens)?, [vec![1, 2, 0], vec![3, 4, 5]]);
845///     assert_eq!(Vec::<i64>::try_from(&lengths)?, [2, 3]);
846///     Ok(())
847/// }
848/// ```
849///
850/// # Infer one sample at a time
851///
852/// [`Self::without_batching`] uses conversion instead of stacking. It adds no
853/// batch dimension and is useful when each dataset item is already a model
854/// input or a preassembled batch. Do not also set `batch_size` or `drop_last`.
855///
856/// ```
857/// use rusttorch_core::Tensor;
858/// use rusttorch_data::{DataLoader, TensorDataset};
859/// fn main() -> Result<(), Box<dyn std::error::Error>> {
860///     let features = Tensor::from_slice(&[1_f32, 2., 3., 4.]).reshape([2, 2]);
861///     let mut loader = DataLoader::builder(TensorDataset::new(vec![features])?)
862///         .without_batching().build()?;
863///     for sample in loader.iter() {
864///         assert_eq!(sample?[0].size(), [2]);
865///     }
866///     Ok(())
867/// }
868/// ```
869///
870/// Use [`Self::dataset_identity`] with a replay-safe dataset to enable exact
871/// restart; [`crate::LoaderState`] has a complete checkpoint example. Independent
872/// stream shards use [`crate::StreamDataLoaderBuilder`] instead.
873pub struct DataLoaderBuilder<
874    D,
875    P,
876    C,
877    F = IdentityTransformFactory,
878    I = NoWorkerInit,
879    X = SerialExecution,
880    M = MemoryDisabled,
881    N = PinDisabled,
882    K = CheckpointDisabled,
883> {
884    dataset: D,
885    plan: P,
886    collator: C,
887    transform_factory: F,
888    worker_init: I,
889    configuration: BuilderConfiguration,
890    explicit: ExplicitArguments,
891    checkpoint: K,
892    states: PhantomData<(X, M, N)>,
893}
894
895impl<D> DataLoaderBuilder<D, AutoBatch<SequentialSampler>, DefaultCollator>
896where
897    D: Dataset,
898{
899    /// Creates the default serial builder for `dataset`.
900    pub fn new(dataset: D) -> Self {
901        Self {
902            plan: AutoBatch {
903                sampler: SequentialSampler::new(dataset.len()),
904                batch_size: NonZeroUsize::MIN,
905                drop_last: false,
906            },
907            dataset,
908            collator: DefaultCollator,
909            transform_factory: IdentityTransformFactory,
910            worker_init: NoWorkerInit,
911            configuration: BuilderConfiguration::default(),
912            explicit: ExplicitArguments::default(),
913            checkpoint: CheckpointDisabled,
914            states: PhantomData,
915        }
916    }
917}
918
919impl<D, P, C, F, I, X, M, N, K> DataLoaderBuilder<D, P, C, F, I, X, M, N, K> {
920    fn map<P2, C2>(
921        self,
922        transform: impl FnOnce(P, C) -> (P2, C2),
923    ) -> DataLoaderBuilder<D, P2, C2, F, I, X, M, N, K> {
924        let (plan, collator) = transform(self.plan, self.collator);
925        DataLoaderBuilder {
926            dataset: self.dataset,
927            plan,
928            collator,
929            transform_factory: self.transform_factory,
930            worker_init: self.worker_init,
931            configuration: self.configuration,
932            explicit: self.explicit,
933            checkpoint: self.checkpoint,
934            states: PhantomData,
935        }
936    }
937
938    /// Sets how many selected samples form one automatic batch; defaults to one.
939    ///
940    /// Zero is rejected by `build()`. The final batch may be smaller unless
941    /// [`Self::drop_last`] is enabled. This option cannot be combined with an
942    /// explicit batch sampler or [`Self::without_batching`].
943    pub fn batch_size(mut self, batch_size: usize) -> Self {
944        self.configuration.batch_size = batch_size;
945        self.explicit.batch_size = true;
946        self
947    }
948
949    /// Omits the final incomplete automatic batch when `true`; defaults to `false`.
950    ///
951    /// Use this when a training step requires a fixed batch size. It does not
952    /// change how a distributed sampler divides indices between ranks.
953    pub fn drop_last(mut self, drop_last: bool) -> Self {
954        self.configuration.drop_last = drop_last;
955        self.explicit.drop_last = true;
956        self
957    }
958
959    /// Selects bounded map-worker execution and its thread-safe capability.
960    ///
961    /// Positive counts share the map dataset through [`Arc`], fetch and
962    /// transform in deterministic worker lanes, and collate on the calling
963    /// thread. Passing zero keeps serial execution, but selecting this method
964    /// still requires worker-safe types at compile time.
965    /// See the [`DataLoaderBuilder`] epoch example for a complete pipeline.
966    pub fn workers(
967        mut self,
968        workers: usize,
969    ) -> DataLoaderBuilder<D, P, C, F, I, WorkerExecution, M, N, K> {
970        self.configuration.workers = workers;
971        DataLoaderBuilder {
972            dataset: self.dataset,
973            plan: self.plan,
974            collator: self.collator,
975            transform_factory: self.transform_factory,
976            worker_init: self.worker_init,
977            configuration: self.configuration,
978            explicit: self.explicit,
979            checkpoint: self.checkpoint,
980            states: PhantomData,
981        }
982    }
983
984    /// Seeds deterministic worker initialization and sample-transform contexts.
985    ///
986    /// This does not enable shuffling or replace the sampler's seed. Select
987    /// [`Self::shuffle`] or a seeded [`Self::sampler`] separately.
988    pub fn seed(mut self, seed: u64) -> Self {
989        self.configuration.loader_seed = seed;
990        self
991    }
992
993    /// Sets the initial epoch used by sampling and task contexts.
994    pub fn epoch(mut self, epoch: u64) -> Self {
995        self.configuration.epoch = epoch;
996        self
997    }
998
999    /// Sets the distributed rank included in deterministic contexts.
1000    ///
1001    /// This does not partition the dataset. Use [`crate::DistributedSampler`]
1002    /// with the same rank to select its indices; gradient synchronization belongs
1003    /// to the training application.
1004    pub fn rank(mut self, rank: usize) -> Self {
1005        self.configuration.rank = rank;
1006        self
1007    }
1008
1009    /// Clones `transform` once for each serial iterator or worker lifecycle.
1010    pub fn transform<T>(
1011        self,
1012        transform: T,
1013    ) -> DataLoaderBuilder<D, P, C, CloneTransformFactory<T>, I, X, M, N, K> {
1014        DataLoaderBuilder {
1015            dataset: self.dataset,
1016            plan: self.plan,
1017            collator: self.collator,
1018            transform_factory: CloneTransformFactory::new(transform),
1019            worker_init: self.worker_init,
1020            configuration: self.configuration,
1021            explicit: self.explicit,
1022            checkpoint: self.checkpoint,
1023            states: PhantomData,
1024        }
1025    }
1026
1027    /// Replaces transform construction with an explicit typed factory.
1028    pub fn transform_factory<F2>(
1029        self,
1030        factory: F2,
1031    ) -> DataLoaderBuilder<D, P, C, F2, I, X, M, N, K> {
1032        DataLoaderBuilder {
1033            dataset: self.dataset,
1034            plan: self.plan,
1035            collator: self.collator,
1036            transform_factory: factory,
1037            worker_init: self.worker_init,
1038            configuration: self.configuration,
1039            explicit: self.explicit,
1040            checkpoint: self.checkpoint,
1041            states: PhantomData,
1042        }
1043    }
1044
1045    /// Marks the current factory's transform as explicitly stateless for
1046    /// exact checkpointing.
1047    pub fn checkpoint_stateless(
1048        self,
1049    ) -> DataLoaderBuilder<D, P, C, StatelessTransformFactory<F>, I, X, M, N, K> {
1050        DataLoaderBuilder {
1051            dataset: self.dataset,
1052            plan: self.plan,
1053            collator: self.collator,
1054            transform_factory: StatelessTransformFactory(self.transform_factory),
1055            worker_init: self.worker_init,
1056            configuration: self.configuration,
1057            explicit: self.explicit,
1058            checkpoint: self.checkpoint,
1059            states: PhantomData,
1060        }
1061    }
1062
1063    /// Marks the current factory's transform as transactional for exact
1064    /// checkpointing.
1065    pub fn checkpoint_transactional(
1066        self,
1067    ) -> DataLoaderBuilder<D, P, C, TransactionalTransformFactory<F>, I, X, M, N, K> {
1068        DataLoaderBuilder {
1069            dataset: self.dataset,
1070            plan: self.plan,
1071            collator: self.collator,
1072            transform_factory: TransactionalTransformFactory(self.transform_factory),
1073            worker_init: self.worker_init,
1074            configuration: self.configuration,
1075            explicit: self.explicit,
1076            checkpoint: self.checkpoint,
1077            states: PhantomData,
1078        }
1079    }
1080
1081    /// Stores the initializer used by future positive-worker execution.
1082    pub fn worker_init<I2>(self, worker_init: I2) -> DataLoaderBuilder<D, P, C, F, I2, X, M, N, K> {
1083        DataLoaderBuilder {
1084            dataset: self.dataset,
1085            plan: self.plan,
1086            collator: self.collator,
1087            transform_factory: self.transform_factory,
1088            worker_init,
1089            configuration: self.configuration,
1090            explicit: self.explicit,
1091            checkpoint: self.checkpoint,
1092            states: PhantomData,
1093        }
1094    }
1095
1096    /// Selects whether positive-worker iterators reuse loader-owned threads.
1097    ///
1098    /// Reuse avoids rebuilding expensive worker-local decoders each epoch.
1099    /// Workers keep their initial [`crate::WorkerInfo`] seeds and factory-created
1100    /// state. Dropping an iterator cancels and waits for its active work; the
1101    /// threads remain available until the loader is dropped and joins the pool.
1102    pub fn persistent_workers(mut self, persistent: bool) -> Self {
1103        self.configuration.persistent_workers = persistent;
1104        self
1105    }
1106
1107    /// Sets the cooperative timeout for each blocking [`Iterator::next`] call.
1108    ///
1109    /// [`Duration::ZERO`] disables the timeout. Expiry cancels the current
1110    /// generation, drains it to quiescence, yields one typed timeout error, and
1111    /// then ends the iterator. User dataset and transform code must observe its
1112    /// [`crate::WorkerContext`] or [`crate::TaskContext`] to stop promptly;
1113    /// Rust threads are never force-cancelled. A nonzero timeout that exceeds
1114    /// the platform monotonic clock range is rejected by [`Self::build`].
1115    pub fn timeout(mut self, timeout: Duration) -> Self {
1116        self.configuration.timeout = (!timeout.is_zero()).then_some(timeout);
1117        self
1118    }
1119
1120    /// Keeps sampler order when `true` (the default), or yields ready batches first.
1121    ///
1122    /// Completion order can reduce waits for a slow batch but changes the order
1123    /// observed by training. Exact checkpointing requires ordered delivery.
1124    pub fn ordered(mut self, ordered: bool) -> Self {
1125        self.configuration.ordered = ordered;
1126        self
1127    }
1128
1129    /// Selects sampler-order or completion-order delivery.
1130    pub fn in_order(self, in_order: bool) -> Self {
1131        self.ordered(in_order)
1132    }
1133
1134    /// Enables automatic recursive pinning for CUDA device zero when available.
1135    ///
1136    /// Pinning happens after collation and preserves the batch type. It prepares
1137    /// host memory for transfer; the application still moves tensors to its
1138    /// model's device. Without CUDA, batches pass through unchanged and
1139    /// [`OwnedDataLoader::pin_memory_status`] reports that pinning is disabled.
1140    ///
1141    /// Enabling pinning requires the final batch type to implement
1142    /// [`PinMemory`] at build time:
1143    ///
1144    /// ```compile_fail
1145    /// use std::convert::Infallible;
1146    /// use rusttorch_data::{DataLoader, Dataset, FnCollate};
1147    ///
1148    /// struct Rows;
1149    /// struct Batch;
1150    /// impl Dataset for Rows {
1151    ///     type Sample = u8;
1152    ///     type Error = Infallible;
1153    ///     fn len(&self) -> usize { 1 }
1154    ///     fn get(&self, _: usize) -> Result<u8, Infallible> { Ok(1) }
1155    /// }
1156    ///
1157    /// let _ = DataLoader::builder(Rows)
1158    ///     .collate(FnCollate::new(|_: Vec<u8>| Ok::<_, Infallible>(Batch)))
1159    ///     .pin_memory()
1160    ///     .build();
1161    /// ```
1162    pub fn pin_memory(mut self) -> DataLoaderBuilder<D, P, C, F, I, X, M, PinEnabled<Auto>, K> {
1163        self.configuration.pin_memory = PinRequest::Auto;
1164        DataLoaderBuilder {
1165            dataset: self.dataset,
1166            plan: self.plan,
1167            collator: self.collator,
1168            transform_factory: self.transform_factory,
1169            worker_init: self.worker_init,
1170            configuration: self.configuration,
1171            explicit: self.explicit,
1172            checkpoint: self.checkpoint,
1173            states: PhantomData,
1174        }
1175    }
1176
1177    /// Enables recursive pinning for one explicit, available CUDA device.
1178    pub fn pin_memory_for(
1179        mut self,
1180        device: Device,
1181    ) -> DataLoaderBuilder<D, P, C, F, I, X, M, PinEnabled<Explicit>, K> {
1182        self.configuration.pin_memory = PinRequest::Explicit(device);
1183        DataLoaderBuilder {
1184            dataset: self.dataset,
1185            plan: self.plan,
1186            collator: self.collator,
1187            transform_factory: self.transform_factory,
1188            worker_init: self.worker_init,
1189            configuration: self.configuration,
1190            explicit: self.explicit,
1191            checkpoint: self.checkpoint,
1192            states: PhantomData,
1193        }
1194    }
1195
1196    /// Bounds outstanding batches per worker; defaults to two with positive workers.
1197    ///
1198    /// For example, four workers and factor two permit eight outstanding batches.
1199    /// Increase this when input preparation cannot otherwise keep training busy;
1200    /// larger values retain more data. The unit here is batches, while
1201    /// [`crate::StreamDataLoaderBuilder::prefetch_factor`] counts records.
1202    ///
1203    /// The worker count multiplied by this factor is checked before any
1204    /// worker starts and is also the global outstanding-work credit limit.
1205    /// Capacities that overflow crossbeam's ring arithmetic or whose
1206    /// conservative aggregate of concrete task/control/completion slots, worker
1207    /// bookkeeping, and channel control-block allowances exceeds the checked
1208    /// 64 MiB queue-allocation ceiling are rejected during build.
1209    pub fn prefetch_factor(mut self, factor: usize) -> Self {
1210        self.configuration.prefetch_factor = Some(factor);
1211        self.explicit.prefetch_factor = true;
1212        self
1213    }
1214
1215    /// Enables a nonzero byte budget for final post-transform prefetched data.
1216    ///
1217    /// The coordinator's active collation batch is outside this queue budget
1218    /// and remains bounded by the configured batch size.
1219    ///
1220    /// Enabling the budget requires the final transformed type to implement
1221    /// [`MemoryFootprint`] at build time:
1222    ///
1223    /// ```compile_fail
1224    /// use std::{convert::Infallible, num::NonZeroUsize};
1225    /// use rusttorch_data::{DataLoader, Dataset, VecCollate};
1226    ///
1227    /// struct Sample;
1228    /// struct Rows;
1229    /// impl Dataset for Rows {
1230    ///     type Sample = Sample;
1231    ///     type Error = Infallible;
1232    ///     fn len(&self) -> usize { 1 }
1233    ///     fn get(&self, _: usize) -> Result<Sample, Infallible> { Ok(Sample) }
1234    /// }
1235    ///
1236    /// let _ = DataLoader::builder(Rows)
1237    ///     .workers(1)
1238    ///     .collate(VecCollate)
1239    ///     .prefetch_bytes(NonZeroUsize::MIN)
1240    ///     .build();
1241    /// ```
1242    pub fn prefetch_bytes(
1243        mut self,
1244        limit: NonZeroUsize,
1245    ) -> DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, N, K> {
1246        self.configuration.prefetch_bytes = Some(limit);
1247        self.explicit.prefetch_bytes = true;
1248        DataLoaderBuilder {
1249            dataset: self.dataset,
1250            plan: self.plan,
1251            collator: self.collator,
1252            transform_factory: self.transform_factory,
1253            worker_init: self.worker_init,
1254            configuration: self.configuration,
1255            explicit: self.explicit,
1256            checkpoint: self.checkpoint,
1257            states: PhantomData,
1258        }
1259    }
1260
1261    /// Replaces automatic batching with explicit reusable index batches.
1262    pub fn batch_sampler<B>(
1263        mut self,
1264        batches: B,
1265    ) -> DataLoaderBuilder<D, ExplicitBatches<B>, C, F, I, X, M, N, K> {
1266        self.explicit.batch_sampler = true;
1267        self.map(|_, collator| (ExplicitBatches { batches }, collator))
1268    }
1269
1270    /// Validates configuration and constructs the owned loader.
1271    ///
1272    /// # Errors
1273    ///
1274    /// Returns [`RustTorchError::InvalidConfiguration`] before plan callbacks
1275    /// for incompatible zero-worker options, zero-valued controls, or
1276    /// unallocatable worker queue sizes.
1277    #[allow(clippy::type_complexity)]
1278    fn build_inner(
1279        mut self,
1280        footprint: Option<FootprintFn<D, F>>,
1281    ) -> Result<OwnedDataLoader<D, P, C, F, I, X, M, N, K>>
1282    where
1283        D: Dataset,
1284        F: TransformFactory<D::Sample>,
1285        P: LoaderPlanConfiguration,
1286        I: WorkerInit,
1287        M: MemoryPolicy,
1288    {
1289        validate_configuration(&self.configuration, self.explicit)?;
1290        let pin_memory_status = resolve_pin_request(self.configuration.pin_memory)?;
1291        let batch_size = validate_batch_size(self.configuration.batch_size)?;
1292        let effective_prefetch = match (
1293            self.configuration.workers,
1294            self.configuration.prefetch_factor,
1295        ) {
1296            (0, _) => None,
1297            (_, Some(factor)) => NonZeroUsize::new(factor),
1298            (_, None) => NonZeroUsize::new(2),
1299        };
1300        let outstanding_capacity = match (self.configuration.workers, effective_prefetch) {
1301            (0, _) => None,
1302            (workers, Some(factor)) => {
1303                Some(workers.checked_mul(factor.get()).ok_or_else(|| {
1304                    invalid_configuration(
1305                        "prefetch_factor",
1306                        "workers multiplied by prefetch_factor exceeds usize",
1307                    )
1308                })?)
1309            }
1310            (_, None) => unreachable!("positive workers always have an effective prefetch factor"),
1311        };
1312        if let Some(capacity) = outstanding_capacity {
1313            validate_worker_pool_capacity::<D, F, I, M>(
1314                self.configuration.workers,
1315                effective_prefetch
1316                    .expect("positive workers have a prefetch factor")
1317                    .get(),
1318                capacity,
1319            )?;
1320        }
1321        self.plan
1322            .apply_batch_options(batch_size, self.configuration.drop_last);
1323        self.plan.apply_epoch(self.configuration.epoch);
1324        Ok(OwnedDataLoader {
1325            dataset: Arc::new(self.dataset),
1326            plan: self.plan,
1327            collator: self.collator,
1328            transform_factory: Arc::new(self.transform_factory),
1329            worker_init: Arc::new(self.worker_init),
1330            workers: self.configuration.workers,
1331            persistent_workers: self.configuration.persistent_workers,
1332            timeout: self.configuration.timeout,
1333            ordered: self.configuration.ordered,
1334            pin_memory_status,
1335            footprint,
1336            effective_prefetch_bytes: self.configuration.prefetch_bytes,
1337            effective_prefetch,
1338            outstanding_capacity,
1339            loader_seed: self.configuration.loader_seed,
1340            rank: self.configuration.rank,
1341            next_generation: 0,
1342            persistent_pool: None,
1343            checkpoint: self.checkpoint,
1344            policies: PhantomData,
1345        })
1346    }
1347}
1348
1349impl<D, P, C, F, I, X, M, N> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointDisabled> {
1350    /// Enables exact checkpointing with a caller-defined dataset identity.
1351    ///
1352    /// The identity must describe the exact dataset contents, not merely its
1353    /// Rust type or path. Empty identities are rejected by [`Self::build`].
1354    /// Wrap an immutable [`crate::ReplaySafeDataset`] in [`crate::ReplaySafeMap`],
1355    /// or use [`crate::TensorDataset::into_replay_safe`] for isolated serial tensor
1356    /// inputs. Built-in samplers, identity transforms and collators support state
1357    /// capture; custom components must provide their checkpoint contracts.
1358    /// [`crate::LoaderState`] shows the complete save/restore flow.
1359    pub fn dataset_identity(
1360        self,
1361        identity: String,
1362    ) -> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointFresh> {
1363        DataLoaderBuilder {
1364            dataset: self.dataset,
1365            plan: self.plan,
1366            collator: self.collator,
1367            transform_factory: self.transform_factory,
1368            worker_init: self.worker_init,
1369            configuration: self.configuration,
1370            explicit: self.explicit,
1371            checkpoint: CheckpointFresh { identity },
1372            states: PhantomData,
1373        }
1374    }
1375}
1376
1377impl<D, P, C, F, I, X, M, N> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointFresh> {
1378    /// Selects a saved map-loader boundary for validated resume.
1379    ///
1380    /// Recreate the dataset and the same loader options, set its identity, then
1381    /// pass a deserialized [`crate::LoaderState`] here. `build()` rejects
1382    /// incompatible schema, identity, configuration or component state before
1383    /// applying a restore. The first `iter()` continues the saved cursor;
1384    /// later calls begin fresh passes. See [`crate::LoaderState`] for an example.
1385    pub fn resume_from<S>(
1386        self,
1387        state: S,
1388    ) -> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointResume<S>> {
1389        DataLoaderBuilder {
1390            dataset: self.dataset,
1391            plan: self.plan,
1392            collator: self.collator,
1393            transform_factory: self.transform_factory,
1394            worker_init: self.worker_init,
1395            configuration: self.configuration,
1396            explicit: self.explicit,
1397            checkpoint: CheckpointResume {
1398                identity: self.checkpoint.identity,
1399                state,
1400            },
1401            states: PhantomData,
1402        }
1403    }
1404}
1405
1406fn resident_bytes<T: MemoryFootprint>(values: &[T]) -> usize {
1407    let mut total = 0usize;
1408    for value in values {
1409        let Some(next) = total.checked_add(value.resident_bytes()) else {
1410            return usize::MAX;
1411        };
1412        total = next;
1413    }
1414    total
1415}
1416
1417impl<D, P, C, F, I, X>
1418    DataLoaderBuilder<D, P, C, F, I, X, MemoryDisabled, PinDisabled, CheckpointDisabled>
1419{
1420    /// Validates configuration and builds an item-bounded, unpinned loader.
1421    #[allow(clippy::type_complexity)]
1422    pub fn build(
1423        self,
1424    ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryDisabled, PinDisabled, CheckpointDisabled>>
1425    where
1426        D: Dataset,
1427        F: TransformFactory<D::Sample>,
1428        P: LoaderPlanConfiguration,
1429        I: WorkerInit,
1430    {
1431        self.build_inner(None)
1432    }
1433}
1434
1435impl<D, P, C, F, I, X>
1436    DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, PinDisabled, CheckpointDisabled>
1437{
1438    /// Validates configuration and builds a byte-bounded, unpinned loader.
1439    #[allow(clippy::type_complexity)]
1440    pub fn build(
1441        self,
1442    ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryEnabled, PinDisabled, CheckpointDisabled>>
1443    where
1444        D: Dataset,
1445        F: TransformFactory<D::Sample>,
1446        TransformOutput<D, F>: MemoryFootprint,
1447        P: LoaderPlanConfiguration,
1448        I: WorkerInit,
1449    {
1450        self.build_inner(Some(resident_bytes::<TransformOutput<D, F>>))
1451    }
1452}
1453
1454impl<D, P, C, F, I, X, Q>
1455    DataLoaderBuilder<D, P, C, F, I, X, MemoryDisabled, PinEnabled<Q>, CheckpointDisabled>
1456{
1457    /// Validates configuration and builds an item-bounded pinned loader.
1458    #[allow(clippy::type_complexity)]
1459    pub fn build(
1460        self,
1461    ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryDisabled, PinEnabled<Q>, CheckpointDisabled>>
1462    where
1463        D: Dataset,
1464        F: TransformFactory<D::Sample>,
1465        P: LoaderPlanConfiguration + LoaderPlan<TransformOutput<D, F>, C>,
1466        P::Batch: PinMemory,
1467        I: WorkerInit,
1468    {
1469        self.build_inner(None)
1470    }
1471}
1472
1473impl<D, P, C, F, I, X, Q>
1474    DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, PinEnabled<Q>, CheckpointDisabled>
1475{
1476    /// Validates configuration and builds a byte-bounded pinned loader.
1477    #[allow(clippy::type_complexity)]
1478    pub fn build(
1479        self,
1480    ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryEnabled, PinEnabled<Q>, CheckpointDisabled>>
1481    where
1482        D: Dataset,
1483        F: TransformFactory<D::Sample>,
1484        TransformOutput<D, F>: MemoryFootprint,
1485        P: LoaderPlanConfiguration + LoaderPlan<TransformOutput<D, F>, C>,
1486        P::Batch: PinMemory,
1487        I: WorkerInit,
1488    {
1489        self.build_inner(Some(resident_bytes::<TransformOutput<D, F>>))
1490    }
1491}
1492
1493#[allow(private_bounds)]
1494impl<D, P, C, F, I, N>
1495    DataLoaderBuilder<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointFresh>
1496where
1497    D: DatasetCheckpoint,
1498    F: TransformFactory<D::Sample>,
1499    F::Transform: WorkerCheckpoint,
1500    P: LoaderPlanConfiguration
1501        + LoaderPlan<TransformOutput<D, F>, C>
1502        + CheckpointPlan<TransformOutput<D, F>, C>,
1503    I: WorkerInit,
1504    N: MapPinPolicy<D, P, C, F>,
1505{
1506    /// Builds a fresh exact serial loader without constructing its transform.
1507    #[allow(clippy::type_complexity)]
1508    pub fn build(
1509        mut self,
1510    ) -> Result<
1511        OwnedDataLoader<
1512            D,
1513            P,
1514            C,
1515            F,
1516            I,
1517            SerialExecution,
1518            MemoryDisabled,
1519            N,
1520            CheckpointActive<P::Iter, F::Transform>,
1521        >,
1522    > {
1523        validate_exact_builder(
1524            &self.configuration,
1525            self.explicit,
1526            &self.checkpoint.identity,
1527        )?;
1528        let pin_memory_status = resolve_pin_request(self.configuration.pin_memory)?;
1529        let batch_size = validate_batch_size(self.configuration.batch_size)?;
1530        self.plan
1531            .apply_batch_options(batch_size, self.configuration.drop_last);
1532        self.plan.apply_epoch(self.configuration.epoch);
1533        let identity = self.plan.checkpoint_identity();
1534        validate_distributed_rank(&identity, self.configuration.rank)?;
1535        let checkpoint_configuration =
1536            checkpoint_configuration(&identity, &self.configuration, pin_memory_status)?;
1537
1538        Ok(OwnedDataLoader {
1539            dataset: Arc::new(self.dataset),
1540            plan: self.plan,
1541            collator: self.collator,
1542            transform_factory: Arc::new(self.transform_factory),
1543            worker_init: Arc::new(self.worker_init),
1544            workers: 0,
1545            persistent_workers: false,
1546            timeout: None,
1547            ordered: true,
1548            pin_memory_status,
1549            footprint: None,
1550            effective_prefetch_bytes: None,
1551            effective_prefetch: None,
1552            outstanding_capacity: None,
1553            loader_seed: self.configuration.loader_seed,
1554            rank: self.configuration.rank,
1555            next_generation: 0,
1556            persistent_pool: None,
1557            checkpoint: CheckpointActive {
1558                identity: self.checkpoint.identity,
1559                configuration: checkpoint_configuration,
1560                pending: None,
1561            },
1562            policies: PhantomData,
1563        })
1564    }
1565}
1566
1567#[allow(private_bounds)]
1568impl<D, P, C, F, I, N, DS, SS, TS, CS>
1569    DataLoaderBuilder<
1570        D,
1571        P,
1572        C,
1573        F,
1574        I,
1575        SerialExecution,
1576        MemoryDisabled,
1577        N,
1578        CheckpointResume<LoaderState<DS, SS, TS, CS>>,
1579    >
1580where
1581    D: DatasetCheckpoint<State = DS>,
1582    F: TransformFactory<D::Sample>,
1583    F::Transform: WorkerCheckpoint<State = TS>,
1584    P: LoaderPlanConfiguration
1585        + LoaderPlan<TransformOutput<D, F>, C>
1586        + CheckpointPlan<TransformOutput<D, F>, C, State = SS, CoordinatorState = CS>,
1587    I: WorkerInit,
1588    N: MapPinPolicy<D, P, C, F>,
1589{
1590    /// Validates and restores an exact serial loader transactionally.
1591    #[allow(clippy::type_complexity)]
1592    pub fn build(
1593        mut self,
1594    ) -> std::result::Result<
1595        OwnedDataLoader<
1596            D,
1597            P,
1598            C,
1599            F,
1600            I,
1601            SerialExecution,
1602            MemoryDisabled,
1603            N,
1604            CheckpointActive<P::Iter, F::Transform>,
1605        >,
1606        CheckpointBuildError<F::Error>,
1607    > {
1608        let CheckpointResume {
1609            identity: requested_identity,
1610            state,
1611        } = self.checkpoint;
1612        validate_exact_builder(&self.configuration, self.explicit, &requested_identity)?;
1613        let pin_memory_status = resolve_pin_request(self.configuration.pin_memory)?;
1614        let batch_size = validate_batch_size(self.configuration.batch_size)?;
1615        validate_static_loader_state_envelope(
1616            &state,
1617            &requested_identity,
1618            &self.configuration,
1619            pin_memory_status,
1620            <P as CheckpointPlan<TransformOutput<D, F>, C>>::AUTOMATIC_BATCHING,
1621            0,
1622        )?;
1623
1624        let plan_identity =
1625            <P as CheckpointPlan<TransformOutput<D, F>, C>>::checkpoint_identity_from_state(
1626                &state.sampler,
1627                batch_size,
1628                self.configuration.drop_last,
1629            )?;
1630        validate_distributed_rank(&plan_identity, self.configuration.rank)?;
1631        let expected_configuration =
1632            checkpoint_configuration(&plan_identity, &self.configuration, pin_memory_status)?;
1633        validate_plan_loader_state_envelope(&state, &expected_configuration)?;
1634
1635        self.dataset.validate_dataset_state(&state.dataset)?;
1636        self.plan.validate_checkpoint_state_with_batch_options(
1637            &state.sampler,
1638            state.epoch,
1639            state.next_batch,
1640            state.next_logical_sample,
1641            batch_size,
1642            self.configuration.drop_last,
1643        )?;
1644        let mut transform = self
1645            .transform_factory
1646            .create(None)
1647            .map_err(CheckpointBuildError::TransformFactory)?;
1648        transform.validate_snapshot(&state.transform)?;
1649        self.plan
1650            .validate_coordinator(&self.collator, &state.collate)?;
1651
1652        self.dataset.restore_dataset_validated(&state.dataset);
1653        self.plan
1654            .apply_batch_options(batch_size, self.configuration.drop_last);
1655        let batches = self.plan.restore_checkpoint_iter_validated(&state.sampler);
1656        transform.restore_validated(&state.transform);
1657        self.plan
1658            .restore_coordinator_validated(&mut self.collator, &state.collate);
1659
1660        Ok(OwnedDataLoader {
1661            dataset: Arc::new(self.dataset),
1662            plan: self.plan,
1663            collator: self.collator,
1664            transform_factory: Arc::new(self.transform_factory),
1665            worker_init: Arc::new(self.worker_init),
1666            workers: 0,
1667            persistent_workers: false,
1668            timeout: None,
1669            ordered: true,
1670            pin_memory_status,
1671            footprint: None,
1672            effective_prefetch_bytes: None,
1673            effective_prefetch: None,
1674            outstanding_capacity: None,
1675            loader_seed: self.configuration.loader_seed,
1676            rank: self.configuration.rank,
1677            next_generation: 0,
1678            persistent_pool: None,
1679            checkpoint: CheckpointActive {
1680                identity: requested_identity,
1681                configuration: expected_configuration,
1682                pending: Some(crate::checkpoint::PendingSerial {
1683                    batches,
1684                    transform,
1685                    next_batch: state.next_batch,
1686                    next_logical_sample: state.next_logical_sample,
1687                }),
1688            },
1689            policies: PhantomData,
1690        })
1691    }
1692}
1693
1694impl<D, S, C, F, I, X, M, N, K> DataLoaderBuilder<D, AutoBatch<S>, C, F, I, X, M, N, K>
1695where
1696    D: Dataset,
1697{
1698    /// Replaces the current sampler.
1699    pub fn sampler<S2>(
1700        mut self,
1701        sampler: S2,
1702    ) -> DataLoaderBuilder<D, AutoBatch<S2>, C, F, I, X, M, N, K> {
1703        self.explicit.sampler = true;
1704        self.map(|plan, collator| {
1705            (
1706                AutoBatch {
1707                    sampler,
1708                    batch_size: plan.batch_size,
1709                    drop_last: plan.drop_last,
1710                },
1711                collator,
1712            )
1713        })
1714    }
1715
1716    /// Replaces the current sampler with a seeded random sampler.
1717    ///
1718    /// # Errors
1719    ///
1720    /// Returns an invalid-configuration error for an empty dataset or an
1721    /// unallocatable permutation.
1722    #[allow(clippy::type_complexity)]
1723    pub fn shuffle(
1724        mut self,
1725        seed: u64,
1726    ) -> Result<DataLoaderBuilder<D, AutoBatch<RandomSampler>, C, F, I, X, M, N, K>> {
1727        self.explicit.shuffle = true;
1728        let sampler = RandomSampler::new(self.dataset.len(), seed)?;
1729        Ok(self.map(|plan, collator| {
1730            (
1731                AutoBatch {
1732                    sampler,
1733                    batch_size: plan.batch_size,
1734                    drop_last: plan.drop_last,
1735                },
1736                collator,
1737            )
1738        }))
1739    }
1740
1741    /// Disables automatic batching and selects default conversion.
1742    pub fn without_batching(
1743        mut self,
1744    ) -> DataLoaderBuilder<D, NoBatch<S, DefaultConverter>, C, F, I, X, M, N, K> {
1745        self.explicit.without_batching = true;
1746        self.map(|plan, collator| {
1747            (
1748                NoBatch {
1749                    sampler: plan.sampler,
1750                    converter: DefaultConverter,
1751                },
1752                collator,
1753            )
1754        })
1755    }
1756
1757    /// Replaces the automatic-batch collator.
1758    pub fn collate<C2>(
1759        self,
1760        collator: C2,
1761    ) -> DataLoaderBuilder<D, AutoBatch<S>, C2, F, I, X, M, N, K> {
1762        self.map(|plan, _| (plan, collator))
1763    }
1764}
1765
1766impl<D, B, C, F, I, X, M, N, K> DataLoaderBuilder<D, ExplicitBatches<B>, C, F, I, X, M, N, K>
1767where
1768    D: Dataset,
1769{
1770    /// Selects a sampler; build rejects its conflict with the earlier batch sampler.
1771    pub fn sampler<S>(
1772        mut self,
1773        sampler: S,
1774    ) -> DataLoaderBuilder<D, AutoBatch<S>, C, F, I, X, M, N, K> {
1775        self.explicit.sampler = true;
1776        let batch_size =
1777            NonZeroUsize::new(self.configuration.batch_size).unwrap_or(NonZeroUsize::MIN);
1778        let drop_last = self.configuration.drop_last;
1779        self.map(|_, collator| {
1780            (
1781                AutoBatch {
1782                    sampler,
1783                    batch_size,
1784                    drop_last,
1785                },
1786                collator,
1787            )
1788        })
1789    }
1790
1791    /// Selects shuffle; build rejects its conflict with the earlier batch sampler.
1792    ///
1793    /// # Errors
1794    ///
1795    /// Returns an invalid-configuration error for an empty dataset or an
1796    /// unallocatable permutation.
1797    #[allow(clippy::type_complexity)]
1798    pub fn shuffle(
1799        mut self,
1800        seed: u64,
1801    ) -> Result<DataLoaderBuilder<D, AutoBatch<RandomSampler>, C, F, I, X, M, N, K>> {
1802        self.explicit.shuffle = true;
1803        let sampler = RandomSampler::new(self.dataset.len(), seed)?;
1804        let batch_size =
1805            NonZeroUsize::new(self.configuration.batch_size).unwrap_or(NonZeroUsize::MIN);
1806        let drop_last = self.configuration.drop_last;
1807        Ok(self.map(|_, collator| {
1808            (
1809                AutoBatch {
1810                    sampler,
1811                    batch_size,
1812                    drop_last,
1813                },
1814                collator,
1815            )
1816        }))
1817    }
1818
1819    /// Disables batching; build rejects its conflict with the earlier batch sampler.
1820    pub fn without_batching(
1821        mut self,
1822    ) -> DataLoaderBuilder<D, NoBatch<SequentialSampler, DefaultConverter>, C, F, I, X, M, N, K>
1823    {
1824        self.explicit.without_batching = true;
1825        let length = self.dataset.len();
1826        self.map(|_, collator| {
1827            (
1828                NoBatch {
1829                    sampler: SequentialSampler::new(length),
1830                    converter: DefaultConverter,
1831                },
1832                collator,
1833            )
1834        })
1835    }
1836
1837    /// Replaces the explicit-batch collator.
1838    pub fn collate<C2>(
1839        self,
1840        collator: C2,
1841    ) -> DataLoaderBuilder<D, ExplicitBatches<B>, C2, F, I, X, M, N, K> {
1842        self.map(|plan, _| (plan, collator))
1843    }
1844}
1845
1846impl<D, S, V, C, F, I, X, M, N, K> DataLoaderBuilder<D, NoBatch<S, V>, C, F, I, X, M, N, K>
1847where
1848    D: Dataset,
1849{
1850    /// Replaces the no-batching sampler.
1851    pub fn sampler<S2>(
1852        mut self,
1853        sampler: S2,
1854    ) -> DataLoaderBuilder<D, NoBatch<S2, V>, C, F, I, X, M, N, K> {
1855        self.explicit.sampler = true;
1856        self.map(|plan, collator| {
1857            (
1858                NoBatch {
1859                    sampler,
1860                    converter: plan.converter,
1861                },
1862                collator,
1863            )
1864        })
1865    }
1866
1867    /// Replaces the no-batching sampler with a seeded random sampler.
1868    ///
1869    /// # Errors
1870    ///
1871    /// Returns an invalid-configuration error for an empty dataset or an
1872    /// unallocatable permutation.
1873    #[allow(clippy::type_complexity)]
1874    pub fn shuffle(
1875        mut self,
1876        seed: u64,
1877    ) -> Result<DataLoaderBuilder<D, NoBatch<RandomSampler, V>, C, F, I, X, M, N, K>> {
1878        self.explicit.shuffle = true;
1879        let sampler = RandomSampler::new(self.dataset.len(), seed)?;
1880        Ok(self.map(|plan, collator| {
1881            (
1882                NoBatch {
1883                    sampler,
1884                    converter: plan.converter,
1885                },
1886                collator,
1887            )
1888        }))
1889    }
1890
1891    /// Replaces the no-batching converter.
1892    pub fn convert<V2>(
1893        self,
1894        converter: V2,
1895    ) -> DataLoaderBuilder<D, NoBatch<S, V2>, C, F, I, X, M, N, K> {
1896        self.map(|plan, collator| {
1897            (
1898                NoBatch {
1899                    sampler: plan.sampler,
1900                    converter,
1901                },
1902                collator,
1903            )
1904        })
1905    }
1906}
1907
1908/// A reusable dataset pipeline, created by [`crate::DataLoader::builder`].
1909///
1910/// Call `iter()` for a pass over the selected epoch and `set_epoch(epoch)` before
1911/// the next epoch's shuffle. The iterator borrows this owner mutably, preventing
1912/// two overlapping passes over the same pipeline. [`Self::len`] reports batches
1913/// (or individual outputs without batching), not the dataset's sample count.
1914///
1915/// The [`DataLoaderBuilder`] examples cover worker prefetch, custom padding and
1916/// inference without batching. Serial loading accepts local non-`Send` types;
1917/// worker loading shares a thread-safe dataset and fetches samples concurrently.
1918/// Dropping an iteration cancels its outstanding work and waits for it to stop.
1919/// Persistent workers remain alive for reuse until this owner is dropped.
1920///
1921/// Exact loaders additionally expose `checkpoint()` on their iterator. Save
1922/// after a successful batch and recreate the matching builder to resume;
1923/// [`crate::LoaderState`] demonstrates the complete sequence.
1924#[allow(private_bounds)]
1925pub struct OwnedDataLoader<
1926    D,
1927    P,
1928    C,
1929    F = IdentityTransformFactory,
1930    I = NoWorkerInit,
1931    X = SerialExecution,
1932    M = MemoryDisabled,
1933    N = PinDisabled,
1934    K = CheckpointDisabled,
1935> where
1936    D: Dataset,
1937    F: TransformFactory<D::Sample>,
1938    I: WorkerInit,
1939    M: MemoryPolicy,
1940{
1941    dataset: Arc<D>,
1942    plan: P,
1943    collator: C,
1944    transform_factory: Arc<F>,
1945    worker_init: Arc<I>,
1946    workers: usize,
1947    persistent_workers: bool,
1948    timeout: Option<Duration>,
1949    ordered: bool,
1950    pin_memory_status: PinMemoryStatus,
1951    footprint: Option<FootprintFn<D, F>>,
1952    effective_prefetch_bytes: Option<NonZeroUsize>,
1953    effective_prefetch: Option<NonZeroUsize>,
1954    outstanding_capacity: Option<usize>,
1955    loader_seed: u64,
1956    rank: usize,
1957    next_generation: u64,
1958    persistent_pool: Option<WorkerPool<D, F, I, M>>,
1959    checkpoint: K,
1960    policies: PhantomData<(X, M, N)>,
1961}
1962
1963#[allow(private_bounds)]
1964impl<D, P, C, F, I, M, N> OwnedDataLoader<D, P, C, F, I, SerialExecution, M, N, CheckpointDisabled>
1965where
1966    D: Dataset,
1967    F: TransformFactory<D::Sample>,
1968    P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
1969    I: WorkerInit,
1970    M: MemoryPolicy,
1971    N: MapPinPolicy<D, P, C, F>,
1972{
1973    /// Starts a fresh finite iteration for the configured epoch.
1974    pub fn iter(&mut self) -> LoaderIter<'_, D, P, C, F, I, M, N, CheckpointDisabled> {
1975        let (transform, transform_error) = match self.transform_factory.create(None) {
1976            Ok(transform) => (Some(transform), None),
1977            Err(error) => (None, Some(error)),
1978        };
1979        let batches = transform.as_ref().map(|_| self.plan.iter());
1980        let epoch = self.plan.epoch();
1981        let run_context = WorkerRunContext::new(0, self.loader_seed, epoch);
1982        LoaderIter {
1983            dataset: self.dataset.as_ref(),
1984            plan: &mut self.plan,
1985            collator: &mut self.collator,
1986            batches,
1987            transform,
1988            transform_error,
1989            next_batch: 0,
1990            next_logical_sample: 0,
1991            exhausted: false,
1992            loader_seed: self.loader_seed,
1993            epoch,
1994            rank: self.rank,
1995            run_context,
1996            pin_memory_status: self.pin_memory_status,
1997            checkpoint: CheckpointDisabled,
1998            output: PhantomData,
1999            policies: PhantomData,
2000        }
2001    }
2002}
2003
2004#[allow(private_bounds, private_interfaces)]
2005impl<D, P, C, F, I, N>
2006    OwnedDataLoader<
2007        D,
2008        P,
2009        C,
2010        F,
2011        I,
2012        SerialExecution,
2013        MemoryDisabled,
2014        N,
2015        CheckpointActive<P::Iter, F::Transform>,
2016    >
2017where
2018    D: DatasetCheckpoint,
2019    F: TransformFactory<D::Sample>,
2020    F::Transform: WorkerCheckpoint,
2021    P: LoaderPlan<TransformOutput<D, F>, C> + CheckpointPlan<TransformOutput<D, F>, C>,
2022    I: WorkerInit,
2023    N: MapPinPolicy<D, P, C, F>,
2024{
2025    /// Starts the retained resumed cursor once, then fresh exact iterations.
2026    pub fn iter(
2027        &mut self,
2028    ) -> LoaderIter<'_, D, P, C, F, I, MemoryDisabled, N, CheckpointIteration> {
2029        let pending = self.checkpoint.pending.take();
2030        let (batches, transform, transform_error, next_batch, next_logical_sample) = match pending {
2031            Some(pending) => (
2032                Some(pending.batches),
2033                Some(pending.transform),
2034                None,
2035                pending.next_batch,
2036                pending.next_logical_sample,
2037            ),
2038            None => match self.transform_factory.create(None) {
2039                Ok(transform) => (Some(self.plan.iter()), Some(transform), None, 0, 0),
2040                Err(error) => (None, None, Some(error), 0, 0),
2041            },
2042        };
2043        let epoch = self.plan.epoch();
2044        let run_context = WorkerRunContext::new(0, self.loader_seed, epoch);
2045        LoaderIter {
2046            dataset: self.dataset.as_ref(),
2047            plan: &mut self.plan,
2048            collator: &mut self.collator,
2049            batches,
2050            transform,
2051            transform_error,
2052            next_batch,
2053            next_logical_sample,
2054            exhausted: false,
2055            loader_seed: self.loader_seed,
2056            epoch,
2057            rank: self.rank,
2058            run_context,
2059            pin_memory_status: self.pin_memory_status,
2060            checkpoint: CheckpointIteration {
2061                identity: self.checkpoint.identity.clone(),
2062                configuration: self.checkpoint.configuration.clone(),
2063                boundary_valid: true,
2064            },
2065            output: PhantomData,
2066            policies: PhantomData,
2067        }
2068    }
2069
2070    /// Selects the epoch used by future iterations.
2071    ///
2072    /// Calling this before the retained resumed cursor is consumed deliberately
2073    /// discards that cursor and its restored transform, so the selected epoch
2074    /// starts as an ordinary fresh iteration instead of mixing two epochs.
2075    pub fn set_epoch(&mut self, epoch: u64) {
2076        self.plan.set_epoch(epoch);
2077        self.checkpoint.pending = None;
2078    }
2079}
2080
2081#[allow(private_bounds)]
2082impl<D, P, C, F, I, X, M, N> OwnedDataLoader<D, P, C, F, I, X, M, N, CheckpointDisabled>
2083where
2084    D: Dataset,
2085    F: TransformFactory<D::Sample>,
2086    P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2087    I: WorkerInit,
2088    M: MemoryPolicy,
2089{
2090    /// Selects the epoch used by future iterations.
2091    pub fn set_epoch(&mut self, epoch: u64) {
2092        self.plan.set_epoch(epoch);
2093    }
2094}
2095
2096#[allow(private_bounds)]
2097impl<D, P, C, F, I, X, M, N, K> OwnedDataLoader<D, P, C, F, I, X, M, N, K>
2098where
2099    D: Dataset,
2100    F: TransformFactory<D::Sample>,
2101    P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2102    I: WorkerInit,
2103    M: MemoryPolicy,
2104{
2105    /// Returns the exact number of yielded items when the source is sized.
2106    pub fn len(&self) -> Option<usize> {
2107        self.plan.exact_len()
2108    }
2109
2110    /// Returns whether iteration is empty when the source is sized.
2111    pub fn is_empty(&self) -> Option<bool> {
2112        self.len().map(|length| length == 0)
2113    }
2114
2115    /// Returns the current epoch.
2116    pub fn epoch(&self) -> u64 {
2117        self.plan.epoch()
2118    }
2119
2120    /// Returns the configured worker count.
2121    pub fn workers(&self) -> usize {
2122        self.workers
2123    }
2124
2125    /// Returns the effective batches-prefetched-per-worker value.
2126    pub fn effective_prefetch_factor(&self) -> Option<NonZeroUsize> {
2127        self.effective_prefetch
2128    }
2129
2130    /// Returns whether results are configured for sampler order.
2131    pub fn is_ordered(&self) -> bool {
2132        self.ordered
2133    }
2134
2135    /// Returns whether pinning was requested.
2136    pub fn pin_memory_enabled(&self) -> bool {
2137        !matches!(self.pin_memory_status, PinMemoryStatus::Disabled)
2138    }
2139
2140    /// Returns the effective recursive pinning behavior.
2141    pub fn pin_memory_status(&self) -> PinMemoryStatus {
2142        self.pin_memory_status
2143    }
2144
2145    /// Returns the effective post-transform prefetch byte budget.
2146    pub fn effective_prefetch_bytes(&self) -> Option<NonZeroUsize> {
2147        self.effective_prefetch_bytes
2148    }
2149
2150    /// Returns the active timeout, if any.
2151    pub fn timeout(&self) -> Option<Duration> {
2152        self.timeout
2153    }
2154
2155    /// Returns whether persistent workers were requested.
2156    pub fn persistent_workers_enabled(&self) -> bool {
2157        self.persistent_workers
2158    }
2159}
2160
2161/// One pass over an [`OwnedDataLoader`], yielding a typed result for each batch.
2162///
2163/// Obtain this from the owner's `iter()` method and use `for batch in ...` or
2164/// `collect::<Result<Vec<_>, _>>()`. Loading, transform and collation failures
2165/// end that pass. A subsequent `iter()` starts a fresh pass; it does not silently
2166/// continue from the failed sample. Checkpoint-enabled iterations can instead
2167/// record a successful boundary with [`Self::checkpoint`].
2168pub struct LoaderIter<
2169    'a,
2170    D,
2171    P,
2172    C,
2173    F = IdentityTransformFactory,
2174    I = NoWorkerInit,
2175    M = MemoryDisabled,
2176    N = PinDisabled,
2177    K = CheckpointDisabled,
2178> where
2179    D: Dataset,
2180    F: TransformFactory<D::Sample>,
2181    P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2182    I: WorkerInit,
2183{
2184    dataset: &'a D,
2185    plan: &'a mut P,
2186    collator: &'a mut C,
2187    batches: Option<P::Iter>,
2188    transform: Option<F::Transform>,
2189    transform_error: Option<F::Error>,
2190    next_batch: u64,
2191    next_logical_sample: u64,
2192    exhausted: bool,
2193    loader_seed: u64,
2194    epoch: u64,
2195    rank: usize,
2196    run_context: WorkerRunContext,
2197    pin_memory_status: PinMemoryStatus,
2198    checkpoint: K,
2199    output: PhantomData<fn() -> I>,
2200    policies: PhantomData<(M, N)>,
2201}
2202
2203#[allow(private_bounds, private_interfaces)]
2204impl<D, P, C, F, I, N> LoaderIter<'_, D, P, C, F, I, MemoryDisabled, N, CheckpointIteration>
2205where
2206    D: DatasetCheckpoint,
2207    F: TransformFactory<D::Sample>,
2208    F::Transform: WorkerCheckpoint,
2209    P: LoaderPlan<TransformOutput<D, F>, C> + CheckpointPlan<TransformOutput<D, F>, C>,
2210    I: WorkerInit,
2211    N: MapPinPolicy<D, P, C, F>,
2212{
2213    /// Captures the exact next-visible-batch boundary.
2214    ///
2215    /// # Errors
2216    ///
2217    /// Returns [`LoaderError::Checkpoint`] after an error, end-of-input, or
2218    /// while a `next` call has advanced work that did not become visible.
2219    #[allow(clippy::type_complexity)]
2220    pub fn checkpoint(
2221        &mut self,
2222    ) -> std::result::Result<
2223        LoaderState<
2224            D::State,
2225            P::State,
2226            <F::Transform as WorkerCheckpoint>::State,
2227            P::CoordinatorState,
2228        >,
2229        IterError<D, P, C, F, I>,
2230    > {
2231        if !self.checkpoint.boundary_valid {
2232            return Err(LoaderError::Checkpoint {
2233                reason: "the active iterator is not at a consumer-visible batch boundary"
2234                    .to_owned(),
2235            });
2236        }
2237        let transform = self
2238            .transform
2239            .as_ref()
2240            .ok_or_else(|| LoaderError::Checkpoint {
2241                reason: "the serial transform was not constructed".to_owned(),
2242            })?;
2243        let sampler = self
2244            .plan
2245            .checkpoint_state(self.next_batch, self.next_logical_sample)
2246            .map_err(LoaderError::Configuration)?;
2247        Ok(LoaderState {
2248            schema_version: LOADER_STATE_SCHEMA_VERSION,
2249            dataset_identity: self.checkpoint.identity.clone(),
2250            epoch: self.epoch,
2251            iterator_generation: 0,
2252            next_batch: self.next_batch,
2253            next_logical_sample: self.next_logical_sample,
2254            dataset: self.dataset.snapshot_dataset(),
2255            sampler,
2256            transform: transform.snapshot(),
2257            collate: self.plan.save_coordinator(self.collator),
2258            rng_derivation_version: TASK_RNG_DERIVATION_VERSION,
2259            worker_seed_derivation_version: WORKER_SEED_DERIVATION_VERSION,
2260            configuration: self.checkpoint.configuration.clone(),
2261        })
2262    }
2263}
2264
2265impl<D, P, C, F, I, M, N, K> Iterator for LoaderIter<'_, D, P, C, F, I, M, N, K>
2266where
2267    D: Dataset,
2268    F: TransformFactory<D::Sample>,
2269    P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2270    I: WorkerInit,
2271    N: MapPinPolicy<D, P, C, F>,
2272    K: SerialBoundaryPolicy,
2273{
2274    type Item = std::result::Result<
2275        P::Batch,
2276        LoaderError<
2277            PipelineError<
2278                D::Error,
2279                <F::Transform as Transform<D::Sample>>::Error,
2280                P::Error,
2281                F::Error,
2282                I::Error,
2283            >,
2284        >,
2285    >;
2286
2287    fn next(&mut self) -> Option<Self::Item> {
2288        if self.exhausted {
2289            return None;
2290        }
2291        self.checkpoint.begin_next();
2292        if let Some(source) = self.transform_error.take() {
2293            self.exhausted = true;
2294            return Some(Err(LoaderError::Pipeline {
2295                batch: None,
2296                worker: None,
2297                source: PipelineError::TransformInit(source),
2298            }));
2299        }
2300
2301        let indices = match self.batches.as_mut().and_then(Iterator::next) {
2302            Some(indices) => indices,
2303            None => {
2304                self.exhausted = true;
2305                return None;
2306            }
2307        };
2308        let batch = self.next_batch;
2309        let samples = match self.dataset.get_batch(&indices) {
2310            Ok(samples) => samples,
2311            Err(source) => {
2312                self.exhausted = true;
2313                return Some(Err(LoaderError::Pipeline {
2314                    batch: Some(batch),
2315                    worker: None,
2316                    source: PipelineError::Dataset(source),
2317                }));
2318            }
2319        };
2320        if samples.len() != indices.len() {
2321            self.exhausted = true;
2322            return Some(Err(LoaderError::InvalidBatchCardinality {
2323                batch,
2324                worker: None,
2325                expected: indices.len(),
2326                actual: samples.len(),
2327            }));
2328        }
2329        let mut transformed = Vec::with_capacity(samples.len());
2330        for sample in samples {
2331            let context = TaskContext {
2332                loader_seed: self.loader_seed,
2333                epoch: self.epoch,
2334                rank: self.rank,
2335                logical_sample: self.next_logical_sample,
2336                stage: 0,
2337                cancellation: self.run_context.cancellation.clone(),
2338                deadline: self.run_context.deadline.clone(),
2339            };
2340            let Some(transform) = self.transform.as_mut() else {
2341                self.exhausted = true;
2342                return None;
2343            };
2344            match transform.transform(sample, &context) {
2345                Ok(sample) => transformed.push(sample),
2346                Err(source) => {
2347                    self.exhausted = true;
2348                    return Some(Err(LoaderError::Pipeline {
2349                        batch: Some(batch),
2350                        worker: None,
2351                        source: PipelineError::Transform(source),
2352                    }));
2353                }
2354            }
2355            match self.next_logical_sample.checked_add(1) {
2356                Some(next) => self.next_logical_sample = next,
2357                None => self.exhausted = true,
2358            }
2359        }
2360        let result =
2361            self.plan
2362                .finish(self.collator, transformed)
2363                .map_err(|source| LoaderError::Pipeline {
2364                    batch: Some(batch),
2365                    worker: None,
2366                    source: PipelineError::Collate(source),
2367                })
2368                .and_then(|value| match self.pin_memory_status {
2369                    PinMemoryStatus::Enabled(device) => N::pin(value, device)
2370                        .map_err(|source| LoaderError::PinMemory { batch, source }),
2371                    PinMemoryStatus::Disabled | PinMemoryStatus::DisabledNoAccelerator => Ok(value),
2372                });
2373        if result.is_err() {
2374            self.exhausted = true;
2375        } else if let Some(next) = self.next_batch.checked_add(1) {
2376            self.next_batch = next;
2377            self.checkpoint.commit_visible();
2378        } else {
2379            self.exhausted = true;
2380        }
2381        Some(result)
2382    }
2383}
2384
2385/// A pass over an owned loader configured through [`DataLoaderBuilder::workers`].
2386///
2387/// The caller receives collated batches while workers prepare bounded pending
2388/// work. Use it like an ordinary fallible iterator; its lifetime keeps the loader
2389/// borrowed until the pass ends. Drop waits for active work to stop, so dataset
2390/// and transform callbacks should cooperate with cancellation. Explicitly
2391/// selecting zero workers uses serial fetching with the same worker-safe types.
2392#[allow(private_bounds)]
2393pub struct WorkerLoaderIter<
2394    'a,
2395    D,
2396    P,
2397    C,
2398    F = IdentityTransformFactory,
2399    I = NoWorkerInit,
2400    M = MemoryDisabled,
2401    N = PinDisabled,
2402    K = CheckpointDisabled,
2403> where
2404    D: Dataset,
2405    F: TransformFactory<D::Sample>,
2406    P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2407    I: WorkerInit,
2408    M: MemoryPolicy,
2409{
2410    dataset: Arc<D>,
2411    plan: &'a mut P,
2412    collator: &'a mut C,
2413    batches: Option<P::Iter>,
2414    serial_transform: Option<F::Transform>,
2415    serial_transform_error: Option<F::Error>,
2416    pool: Option<IteratorPool<'a, D, F, I, M>>,
2417    completed: BTreeMap<u64, CompletedBatch<D, F, M>>,
2418    pending_error: Option<IterError<D, P, C, F, I>>,
2419    generation: u64,
2420    run_context: Option<WorkerRunContext>,
2421    workers: usize,
2422    capacity: usize,
2423    ordered: bool,
2424    timeout: Option<Duration>,
2425    loader_seed: u64,
2426    epoch: u64,
2427    rank: usize,
2428    next_submission: u64,
2429    next_visible: u64,
2430    next_logical_sample: u64,
2431    pending_submission: Option<PendingSubmission>,
2432    outstanding: usize,
2433    source_exhausted: bool,
2434    submission_closed: bool,
2435    exhausted: bool,
2436    pin_memory_status: PinMemoryStatus,
2437    _checkpoint: K,
2438    policies: PhantomData<(M, N)>,
2439}
2440
2441enum IteratorPool<'a, D, F, I, M>
2442where
2443    D: Dataset,
2444    F: TransformFactory<D::Sample>,
2445    I: WorkerInit,
2446    M: MemoryPolicy,
2447{
2448    Owned(WorkerPool<D, F, I, M>),
2449    Persistent {
2450        pool: &'a mut WorkerPool<D, F, I, M>,
2451        generation: u64,
2452    },
2453}
2454
2455impl<D, F, I, M> IteratorPool<'_, D, F, I, M>
2456where
2457    D: Dataset,
2458    F: TransformFactory<D::Sample>,
2459    I: WorkerInit,
2460    M: MemoryPolicy,
2461{
2462    fn pool(&self) -> &WorkerPool<D, F, I, M> {
2463        match self {
2464            Self::Owned(pool) => pool,
2465            Self::Persistent { pool, .. } => pool,
2466        }
2467    }
2468
2469    fn pool_mut(&mut self) -> &mut WorkerPool<D, F, I, M> {
2470        match self {
2471            Self::Owned(pool) => pool,
2472            Self::Persistent { pool, .. } => pool,
2473        }
2474    }
2475
2476    fn is_persistent(&self) -> bool {
2477        matches!(self, Self::Persistent { .. })
2478    }
2479}
2480
2481impl<D, F, I, M> Drop for IteratorPool<'_, D, F, I, M>
2482where
2483    D: Dataset,
2484    F: TransformFactory<D::Sample>,
2485    I: WorkerInit,
2486    M: MemoryPolicy,
2487{
2488    fn drop(&mut self) {
2489        if let Self::Persistent { pool, generation } = self
2490            && pool.quiesce(*generation).is_err()
2491        {
2492            pool.poison();
2493        }
2494    }
2495}
2496
2497#[allow(private_bounds)]
2498impl<D, P, C, F, I, M, N> OwnedDataLoader<D, P, C, F, I, WorkerExecution, M, N, CheckpointDisabled>
2499where
2500    D: Dataset + Send + Sync + 'static,
2501    D::Sample: Send + 'static,
2502    D::Error: Send + 'static,
2503    F: TransformFactory<D::Sample> + Send + Sync + 'static,
2504    F::Transform: Send + 'static,
2505    <F::Transform as Transform<D::Sample>>::Output: Send + 'static,
2506    <F::Transform as Transform<D::Sample>>::Error: Send + 'static,
2507    F::Error: Send + 'static,
2508    P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2509    I: WorkerInit + Send + Sync + 'static,
2510    I::Error: Send + 'static,
2511    M: MemoryPolicy + Send + 'static,
2512    N: MapPinPolicy<D, P, C, F>,
2513{
2514    /// Starts a fresh generation on a bounded worker pool, or the explicit
2515    /// zero-worker path.
2516    pub fn iter(&mut self) -> WorkerLoaderIter<'_, D, P, C, F, I, M, N, CheckpointDisabled> {
2517        let (epoch, epoch_panicked) = if self.workers == 0 {
2518            (self.plan.epoch(), false)
2519        } else {
2520            match catch_unwind(AssertUnwindSafe(|| self.plan.epoch())) {
2521                Ok(epoch) => (epoch, false),
2522                Err(_) => (0, true),
2523            }
2524        };
2525        let mut iterator = WorkerLoaderIter {
2526            dataset: Arc::clone(&self.dataset),
2527            plan: &mut self.plan,
2528            collator: &mut self.collator,
2529            batches: None,
2530            serial_transform: None,
2531            serial_transform_error: None,
2532            pool: None,
2533            completed: BTreeMap::new(),
2534            pending_error: None,
2535            generation: self.next_generation,
2536            run_context: Some(WorkerRunContext::new(
2537                self.next_generation,
2538                self.loader_seed,
2539                epoch,
2540            )),
2541            workers: self.workers,
2542            capacity: self.outstanding_capacity.unwrap_or(0),
2543            ordered: self.ordered,
2544            timeout: self.timeout,
2545            loader_seed: self.loader_seed,
2546            epoch,
2547            rank: self.rank,
2548            next_submission: 0,
2549            next_visible: 0,
2550            next_logical_sample: 0,
2551            pending_submission: None,
2552            outstanding: 0,
2553            source_exhausted: false,
2554            submission_closed: false,
2555            exhausted: false,
2556            pin_memory_status: self.pin_memory_status,
2557            _checkpoint: CheckpointDisabled,
2558            policies: PhantomData,
2559        };
2560
2561        if epoch_panicked {
2562            iterator.pending_error = Some(LoaderError::CoordinatorPanic {
2563                stage: P::EPOCH_PANIC_STAGE,
2564                batch: None,
2565            });
2566            return iterator;
2567        }
2568        if self.workers == 0 {
2569            match self.transform_factory.create(None) {
2570                Ok(transform) => {
2571                    iterator.serial_transform = Some(transform);
2572                    iterator.batches = Some(iterator.plan.iter());
2573                }
2574                Err(error) => iterator.serial_transform_error = Some(error),
2575            }
2576            return iterator;
2577        }
2578        let Some(next_generation) = self.next_generation.checked_add(1) else {
2579            iterator.pending_error = Some(LoaderError::Configuration(invalid_configuration(
2580                "workers",
2581                "iterator generation overflowed",
2582            )));
2583            return iterator;
2584        };
2585        iterator.batches = match catch_unwind(AssertUnwindSafe(|| iterator.plan.iter())) {
2586            Ok(batches) => Some(batches),
2587            Err(_) => {
2588                iterator.pending_error = Some(LoaderError::CoordinatorPanic {
2589                    stage: P::CREATION_PANIC_STAGE,
2590                    batch: None,
2591                });
2592                return iterator;
2593            }
2594        };
2595        self.next_generation = next_generation;
2596        let configuration = WorkerPoolConfiguration {
2597            workers: self.workers,
2598            prefetch_factor: self
2599                .effective_prefetch
2600                .expect("positive workers have a validated prefetch factor")
2601                .get(),
2602            result_capacity: iterator.capacity,
2603            seed_generation: iterator.generation,
2604            loader_seed: self.loader_seed,
2605            rank: self.rank,
2606        };
2607        let run_context =
2608            WorkerRunContext::new(iterator.generation, self.loader_seed, iterator.epoch)
2609                .with_byte_budget(
2610                    self.effective_prefetch_bytes
2611                        .map(|limit| ByteBudget::new(limit.get())),
2612                );
2613        iterator.run_context = Some(run_context.clone());
2614
2615        if self.persistent_workers {
2616            if self.persistent_pool.is_none() {
2617                match WorkerPool::new(
2618                    Arc::clone(&self.dataset),
2619                    Arc::clone(&self.transform_factory),
2620                    Arc::clone(&self.worker_init),
2621                    configuration,
2622                    self.footprint,
2623                    self.ordered,
2624                ) {
2625                    Ok(pool) => self.persistent_pool = Some(pool),
2626                    Err(error) => {
2627                        iterator.pending_error = Some(LoaderError::Configuration(error));
2628                        return iterator;
2629                    }
2630                }
2631            }
2632            let pool = self
2633                .persistent_pool
2634                .as_mut()
2635                .expect("persistent pool was created");
2636            if pool.is_poisoned() || pool.start_generation(run_context).is_err() {
2637                iterator.pending_error = Some(LoaderError::ChannelClosed { batch: 0 });
2638                return iterator;
2639            }
2640            iterator.pool = Some(IteratorPool::Persistent {
2641                pool,
2642                generation: iterator.generation,
2643            });
2644        } else {
2645            match WorkerPool::new(
2646                Arc::clone(&self.dataset),
2647                Arc::clone(&self.transform_factory),
2648                Arc::clone(&self.worker_init),
2649                configuration,
2650                self.footprint,
2651                self.ordered,
2652            ) {
2653                Ok(mut pool) => {
2654                    if pool.start_generation(run_context).is_err() {
2655                        iterator.pending_error = Some(LoaderError::ChannelClosed { batch: 0 });
2656                    } else {
2657                        iterator.pool = Some(IteratorPool::Owned(pool));
2658                    }
2659                }
2660                Err(error) => iterator.pending_error = Some(LoaderError::Configuration(error)),
2661            }
2662        }
2663        if iterator.pending_error.is_none() {
2664            iterator.fill_available();
2665        }
2666        iterator
2667    }
2668}
2669
2670#[allow(private_bounds)]
2671impl<D, P, C, F, I, M, N, K> WorkerLoaderIter<'_, D, P, C, F, I, M, N, K>
2672where
2673    D: Dataset + Send + Sync + 'static,
2674    D::Sample: Send + 'static,
2675    D::Error: Send + 'static,
2676    F: TransformFactory<D::Sample> + Send + Sync + 'static,
2677    F::Transform: Send + 'static,
2678    <F::Transform as Transform<D::Sample>>::Output: Send + 'static,
2679    <F::Transform as Transform<D::Sample>>::Error: Send + 'static,
2680    F::Error: Send + 'static,
2681    P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2682    I: WorkerInit + Send + Sync + 'static,
2683    I::Error: Send + 'static,
2684    M: MemoryPolicy + Send + 'static,
2685    N: MapPinPolicy<D, P, C, F>,
2686    K: WorkerBoundaryPolicy<IterError<D, P, C, F, I>>,
2687{
2688    fn fill_available(&mut self) {
2689        while self.outstanding < self.capacity && !self.source_exhausted && !self.submission_closed
2690        {
2691            if !self.submit_one() {
2692                break;
2693            }
2694        }
2695    }
2696
2697    fn submit_one(&mut self) -> bool {
2698        if let Some(pending) = self.pending_submission.take() {
2699            return self.try_submit(pending);
2700        }
2701        let next_batch = catch_unwind(AssertUnwindSafe(|| {
2702            self.batches.as_mut().and_then(Iterator::next)
2703        }));
2704        let Some(indices) = (match next_batch {
2705            Ok(indices) => indices,
2706            Err(_) => {
2707                self.pending_error = Some(LoaderError::CoordinatorPanic {
2708                    stage: P::REFILL_PANIC_STAGE,
2709                    batch: Some(self.next_submission),
2710                });
2711                self.submission_closed = true;
2712                return false;
2713            }
2714        }) else {
2715            self.source_exhausted = true;
2716            return false;
2717        };
2718        let batch_sequence = self.next_submission;
2719        let Some(next_submission) = batch_sequence.checked_add(1) else {
2720            self.pending_error = Some(LoaderError::Configuration(invalid_configuration(
2721                "sampler",
2722                "batch sequence overflowed",
2723            )));
2724            self.submission_closed = true;
2725            return false;
2726        };
2727        let logical_start = self.next_logical_sample;
2728        let Some(logical_end) = logical_start.checked_add(indices.len() as u64) else {
2729            self.pending_error = Some(LoaderError::Configuration(invalid_configuration(
2730                "sampler",
2731                "logical sample identifier overflowed",
2732            )));
2733            self.submission_closed = true;
2734            return false;
2735        };
2736        let logical_samples = (logical_start..logical_end).collect();
2737        let worker = batch_sequence as usize % self.workers;
2738        self.try_submit(PendingSubmission {
2739            worker,
2740            task: WorkerTask {
2741                generation: self.generation,
2742                batch_sequence,
2743                logical_samples,
2744                indices,
2745            },
2746            next_submission,
2747            logical_end,
2748        })
2749    }
2750
2751    fn try_submit(&mut self, pending: PendingSubmission) -> bool {
2752        let PendingSubmission {
2753            worker,
2754            task,
2755            next_submission,
2756            logical_end,
2757        } = pending;
2758        match self
2759            .pool
2760            .as_ref()
2761            .expect("submissions require an active worker pool")
2762            .pool()
2763            .submit(worker, task)
2764        {
2765            WorkerSubmit::Submitted => {
2766                self._checkpoint.submitted(next_submission - 1, logical_end);
2767                self.next_submission = next_submission;
2768                self.next_logical_sample = logical_end;
2769                self.outstanding += 1;
2770                true
2771            }
2772            WorkerSubmit::Full(task) => {
2773                self.pending_submission = Some(PendingSubmission {
2774                    worker,
2775                    task,
2776                    next_submission,
2777                    logical_end,
2778                });
2779                false
2780            }
2781            WorkerSubmit::Closed => {
2782                self.submission_closed = true;
2783                false
2784            }
2785        }
2786    }
2787
2788    fn stop(&mut self) {
2789        self.stop_inner(false);
2790    }
2791
2792    fn stop_poisoned(&mut self) {
2793        self.stop_inner(true);
2794    }
2795
2796    fn stop_inner(&mut self, poisoned: bool) {
2797        self.exhausted = true;
2798        if let Some(run_context) = &self.run_context {
2799            run_context.cancel();
2800        }
2801        if let Some(mut pool) = self.pool.take() {
2802            let should_poison = poisoned
2803                || (pool.is_persistent() && pool.pool_mut().quiesce(self.generation).is_err());
2804            if should_poison {
2805                pool.pool_mut().poison();
2806            }
2807        }
2808        self.completed.clear();
2809        self.pending_submission = None;
2810    }
2811
2812    fn next_serial(&mut self) -> Option<IterResult<D, P, C, F, I>> {
2813        if let Some(source) = self.serial_transform_error.take() {
2814            self.exhausted = true;
2815            return Some(Err(LoaderError::Pipeline {
2816                batch: None,
2817                worker: None,
2818                source: PipelineError::TransformInit(source),
2819            }));
2820        }
2821        let indices = match self.batches.as_mut().and_then(Iterator::next) {
2822            Some(indices) => indices,
2823            None => {
2824                self.exhausted = true;
2825                return None;
2826            }
2827        };
2828        let batch = self.next_visible;
2829        let samples = match self.dataset.get_batch(&indices) {
2830            Ok(samples) => samples,
2831            Err(source) => {
2832                self.exhausted = true;
2833                return Some(Err(LoaderError::Pipeline {
2834                    batch: Some(batch),
2835                    worker: None,
2836                    source: PipelineError::Dataset(source),
2837                }));
2838            }
2839        };
2840        if samples.len() != indices.len() {
2841            self.exhausted = true;
2842            return Some(Err(LoaderError::InvalidBatchCardinality {
2843                batch,
2844                worker: None,
2845                expected: indices.len(),
2846                actual: samples.len(),
2847            }));
2848        }
2849        let mut transformed = Vec::with_capacity(samples.len());
2850        for sample in samples {
2851            let context = TaskContext {
2852                loader_seed: self.loader_seed,
2853                epoch: self.epoch,
2854                rank: self.rank,
2855                logical_sample: self.next_logical_sample,
2856                stage: 0,
2857                cancellation: self
2858                    .run_context
2859                    .as_ref()
2860                    .expect("serial generation context")
2861                    .cancellation
2862                    .clone(),
2863                deadline: self
2864                    .run_context
2865                    .as_ref()
2866                    .expect("serial generation context")
2867                    .deadline
2868                    .clone(),
2869            };
2870            let transform = self
2871                .serial_transform
2872                .as_mut()
2873                .expect("serial transform exists after successful construction");
2874            match transform.transform(sample, &context) {
2875                Ok(sample) => transformed.push(sample),
2876                Err(source) => {
2877                    self.exhausted = true;
2878                    return Some(Err(LoaderError::Pipeline {
2879                        batch: Some(batch),
2880                        worker: None,
2881                        source: PipelineError::Transform(source),
2882                    }));
2883                }
2884            }
2885            self.next_logical_sample += 1;
2886        }
2887        let result =
2888            self.plan
2889                .finish(self.collator, transformed)
2890                .map_err(|source| LoaderError::Pipeline {
2891                    batch: Some(batch),
2892                    worker: None,
2893                    source: PipelineError::Collate(source),
2894                })
2895                .and_then(|value| match self.pin_memory_status {
2896                    PinMemoryStatus::Enabled(device) => N::pin(value, device)
2897                        .map_err(|source| LoaderError::PinMemory { batch, source }),
2898                    PinMemoryStatus::Disabled | PinMemoryStatus::DisabledNoAccelerator => Ok(value),
2899                });
2900        if result.is_err() {
2901            self.exhausted = true;
2902        } else {
2903            self._checkpoint
2904                .committed(self.next_visible, self.next_logical_sample);
2905            self.next_visible += 1;
2906        }
2907        Some(result)
2908    }
2909
2910    fn publish(
2911        &mut self,
2912        worker: usize,
2913        batch: WorkerBatch<
2914            <F::Transform as Transform<D::Sample>>::Output,
2915            <M as MemoryPolicy>::Permit,
2916        >,
2917    ) -> IterResult<D, P, C, F, I> {
2918        debug_assert_eq!(batch.generation, self.generation);
2919        self.outstanding -= 1;
2920        let sequence = batch.batch_sequence;
2921        let WorkerBatch {
2922            batch_sequence,
2923            samples,
2924            permit,
2925            ..
2926        } = batch;
2927        drop(permit);
2928        let result = match catch_unwind(AssertUnwindSafe(|| {
2929            self.plan.finish(self.collator, samples)
2930        })) {
2931            Ok(result) => result.map_err(|source| LoaderError::Pipeline {
2932                batch: Some(sequence),
2933                worker: None,
2934                source: PipelineError::Collate(source),
2935            }),
2936            Err(_) => Err(LoaderError::CoordinatorPanic {
2937                stage: P::FINISH_PANIC_STAGE,
2938                batch: Some(sequence),
2939            }),
2940        };
2941        let result = result.and_then(|value| match self.pin_memory_status {
2942            PinMemoryStatus::Enabled(device) => {
2943                N::pin(value, device).map_err(|source| LoaderError::PinMemory {
2944                    batch: batch_sequence,
2945                    source,
2946                })
2947            }
2948            PinMemoryStatus::Disabled | PinMemoryStatus::DisabledNoAccelerator => Ok(value),
2949        });
2950        if result.is_ok() {
2951            self._checkpoint
2952                .committed(sequence, self.next_logical_sample);
2953            self.next_visible = self.next_visible.saturating_add(1);
2954            self.fill_available();
2955        } else {
2956            self.stop();
2957        }
2958        let _ = worker;
2959        result
2960    }
2961
2962    fn map_failure(
2963        &mut self,
2964        worker: usize,
2965        batch: Option<u64>,
2966        failure: WorkerStageFailure<D, F, I>,
2967    ) -> IterError<D, P, C, F, I> {
2968        match failure {
2969            WorkerFailure::Dataset(source) => LoaderError::Pipeline {
2970                batch,
2971                worker: Some(worker),
2972                source: PipelineError::Dataset(source),
2973            },
2974            WorkerFailure::Transform(source) => LoaderError::Pipeline {
2975                batch,
2976                worker: Some(worker),
2977                source: PipelineError::Transform(source),
2978            },
2979            WorkerFailure::TransformInit(source) => LoaderError::Pipeline {
2980                batch,
2981                worker: Some(worker),
2982                source: PipelineError::TransformInit(source),
2983            },
2984            WorkerFailure::WorkerInit(source) => LoaderError::Pipeline {
2985                batch,
2986                worker: Some(worker),
2987                source: PipelineError::WorkerInit(source),
2988            },
2989            WorkerFailure::InvalidBatchCardinality { expected, actual } => {
2990                LoaderError::InvalidBatchCardinality {
2991                    batch: batch.unwrap_or(self.next_visible),
2992                    worker: Some(worker),
2993                    expected,
2994                    actual,
2995                }
2996            }
2997            WorkerFailure::MemoryLimit { limit, actual } => LoaderError::MemoryLimit {
2998                batch,
2999                worker: Some(worker),
3000                sequence: None,
3001                logical_id: None,
3002                limit,
3003                actual,
3004            },
3005            WorkerFailure::Panic => LoaderError::WorkerPanic { worker, batch },
3006        }
3007    }
3008}
3009
3010impl<D, P, C, F, I, M, N, K> Iterator for WorkerLoaderIter<'_, D, P, C, F, I, M, N, K>
3011where
3012    D: Dataset + Send + Sync + 'static,
3013    D::Sample: Send + 'static,
3014    D::Error: Send + 'static,
3015    F: TransformFactory<D::Sample> + Send + Sync + 'static,
3016    F::Transform: Send + 'static,
3017    <F::Transform as Transform<D::Sample>>::Output: Send + 'static,
3018    <F::Transform as Transform<D::Sample>>::Error: Send + 'static,
3019    F::Error: Send + 'static,
3020    P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
3021    I: WorkerInit + Send + Sync + 'static,
3022    I::Error: Send + 'static,
3023    M: MemoryPolicy + Send + 'static,
3024    N: MapPinPolicy<D, P, C, F>,
3025    K: WorkerBoundaryPolicy<IterError<D, P, C, F, I>>,
3026{
3027    type Item = std::result::Result<
3028        P::Batch,
3029        LoaderError<
3030            PipelineError<
3031                D::Error,
3032                <F::Transform as Transform<D::Sample>>::Error,
3033                P::Error,
3034                F::Error,
3035                I::Error,
3036            >,
3037        >,
3038    >;
3039
3040    fn next(&mut self) -> Option<Self::Item> {
3041        self._checkpoint.begin_next();
3042        if self.exhausted {
3043            return None;
3044        }
3045        if let Some(error) = self.pending_error.take() {
3046            self.stop();
3047            return Some(Err(error));
3048        }
3049        if self.workers == 0 {
3050            return self.next_serial();
3051        }
3052        let mut deadline_armed = false;
3053        loop {
3054            if let Some(error) = self._checkpoint.take_error(self.next_visible) {
3055                self.stop();
3056                return Some(Err(error));
3057            }
3058            if self.ordered
3059                && let Some((worker, batch)) = self.completed.remove(&self.next_visible)
3060            {
3061                if deadline_armed {
3062                    self.run_context
3063                        .as_ref()
3064                        .expect("worker context")
3065                        .deadline
3066                        .disarm();
3067                }
3068                return Some(self.publish(worker, batch));
3069            }
3070            if self.outstanding == 0 && self.source_exhausted {
3071                if deadline_armed {
3072                    self.run_context
3073                        .as_ref()
3074                        .expect("worker context")
3075                        .deadline
3076                        .disarm();
3077                }
3078                self.stop();
3079                return None;
3080            }
3081            let run_context = self
3082                .run_context
3083                .as_ref()
3084                .expect("worker iteration has a generation context")
3085                .clone();
3086            if !deadline_armed && let Some(timeout) = self.timeout {
3087                run_context.deadline.arm(timeout);
3088                deadline_armed = true;
3089            }
3090            let remaining = self.timeout.and_then(|_| run_context.deadline.remaining());
3091            let received = self
3092                .pool
3093                .as_ref()
3094                .expect("worker iteration has an active pool")
3095                .pool()
3096                .receive(&run_context, remaining);
3097            let completion = match received {
3098                WorkerReceive::Completion(completion) => completion,
3099                WorkerReceive::Timeout => {
3100                    let batch = self.next_visible;
3101                    run_context.cancel();
3102                    run_context.deadline.disarm();
3103                    self.stop();
3104                    return Some(Err(LoaderError::Timeout { batch }));
3105                }
3106                WorkerReceive::Cancelled => {
3107                    run_context.deadline.disarm();
3108                    self.stop();
3109                    return Some(Err(LoaderError::Cancelled));
3110                }
3111                WorkerReceive::Closed => {
3112                    let batch = self.next_visible;
3113                    run_context.deadline.disarm();
3114                    self.stop_poisoned();
3115                    return Some(Err(LoaderError::ChannelClosed { batch }));
3116                }
3117            };
3118            if completion.generation != self.generation {
3119                continue;
3120            }
3121            match completion.result {
3122                Err(failure) => {
3123                    let poisons_pool = matches!(
3124                        &failure,
3125                        WorkerFailure::TransformInit(_)
3126                            | WorkerFailure::WorkerInit(_)
3127                            | WorkerFailure::Panic
3128                    );
3129                    let error =
3130                        self.map_failure(completion.worker, completion.batch_sequence, failure);
3131                    if !poisons_pool && let Some(sequence) = completion.batch_sequence {
3132                        match self._checkpoint.defer_error(sequence, error) {
3133                            None => continue,
3134                            Some(error) => {
3135                                run_context.deadline.disarm();
3136                                self.stop();
3137                                return Some(Err(error));
3138                            }
3139                        }
3140                    }
3141                    run_context.deadline.disarm();
3142                    if poisons_pool {
3143                        self.stop_poisoned();
3144                    } else {
3145                        self.stop();
3146                    }
3147                    return Some(Err(error));
3148                }
3149                Ok(WorkerMessage::Batch(batch))
3150                    if self.ordered && batch.batch_sequence != self.next_visible =>
3151                {
3152                    self.completed
3153                        .insert(batch.batch_sequence, (completion.worker, batch));
3154                }
3155                Ok(WorkerMessage::Batch(batch)) => {
3156                    run_context.deadline.disarm();
3157                    return Some(self.publish(completion.worker, batch));
3158                }
3159                Ok(WorkerMessage::Quiesced) => continue,
3160            }
3161        }
3162    }
3163}
3164
3165fn validate_configuration(
3166    configuration: &BuilderConfiguration,
3167    explicit: ExplicitArguments,
3168) -> Result<()> {
3169    if configuration.batch_size == 0 {
3170        return Err(invalid_configuration(
3171            "batch_size",
3172            "must be greater than zero",
3173        ));
3174    }
3175    if explicit.sampler && explicit.shuffle {
3176        return Err(invalid_configuration(
3177            "sampler",
3178            "cannot be combined with shuffle",
3179        ));
3180    }
3181    if explicit.batch_sampler
3182        && (explicit.batch_size
3183            || explicit.shuffle
3184            || explicit.sampler
3185            || explicit.drop_last
3186            || explicit.without_batching)
3187    {
3188        return Err(invalid_configuration(
3189            "batch_sampler",
3190            "cannot be combined with batch_size, shuffle, sampler, drop_last, or no batching",
3191        ));
3192    }
3193    if explicit.without_batching && explicit.drop_last && configuration.drop_last {
3194        return Err(invalid_configuration(
3195            "drop_last",
3196            "cannot be enabled when automatic batching is disabled",
3197        ));
3198    }
3199    if configuration.workers == 0 && configuration.timeout.is_some() {
3200        return Err(invalid_configuration(
3201            "timeout",
3202            "requires a positive worker count",
3203        ));
3204    }
3205    if configuration
3206        .timeout
3207        .is_some_and(|timeout| !Deadline::can_represent(timeout))
3208    {
3209        return Err(invalid_configuration(
3210            "timeout",
3211            "exceeds the platform monotonic clock range",
3212        ));
3213    }
3214    if explicit.prefetch_factor && configuration.workers == 0 {
3215        return Err(invalid_configuration(
3216            "prefetch_factor",
3217            "requires a positive worker count",
3218        ));
3219    }
3220    if explicit.prefetch_bytes && configuration.workers == 0 {
3221        return Err(invalid_configuration(
3222            "prefetch_bytes",
3223            "requires a positive worker count",
3224        ));
3225    }
3226    if configuration.prefetch_factor == Some(0) {
3227        return Err(invalid_configuration(
3228            "prefetch_factor",
3229            "must be greater than zero",
3230        ));
3231    }
3232    if configuration.persistent_workers && configuration.workers == 0 {
3233        return Err(invalid_configuration(
3234            "persistent_workers",
3235            "requires a positive worker count",
3236        ));
3237    }
3238    Ok(())
3239}
3240
3241fn validate_exact_builder(
3242    configuration: &BuilderConfiguration,
3243    explicit: ExplicitArguments,
3244    identity: &str,
3245) -> Result<()> {
3246    validate_configuration(configuration, explicit)?;
3247    if identity.is_empty() {
3248        return Err(invalid_configuration(
3249            "dataset_identity",
3250            "must not be empty",
3251        ));
3252    }
3253    if configuration.workers != 0 {
3254        return Err(invalid_configuration(
3255            "workers",
3256            "Task 11 exact checkpoints require serial execution",
3257        ));
3258    }
3259    if !configuration.ordered {
3260        return Err(invalid_configuration(
3261            "in_order",
3262            "exact checkpoints require ordered delivery",
3263        ));
3264    }
3265    Ok(())
3266}
3267
3268fn validate_distributed_rank(identity: &PlanCheckpointIdentity, rank: usize) -> Result<()> {
3269    if let Some(distributed) = identity.distributed
3270        && distributed.rank != rank
3271    {
3272        return Err(invalid_configuration(
3273            "rank",
3274            "loader task rank must equal the distributed sampler rank",
3275        ));
3276    }
3277    Ok(())
3278}
3279
3280fn checkpoint_configuration(
3281    identity: &PlanCheckpointIdentity,
3282    builder: &BuilderConfiguration,
3283    pin_status: PinMemoryStatus,
3284) -> Result<crate::LoaderConfiguration> {
3285    let (replicas, distributed_shuffle, distributed_seed, distributed_drop_last) =
3286        match identity.distributed {
3287            Some(distributed) => (
3288                distributed.replicas,
3289                Some(distributed.shuffle),
3290                Some(distributed.seed),
3291                Some(distributed.drop_last),
3292            ),
3293            None => (1, None, None, None),
3294        };
3295    let pin_request = checkpoint_pin_request(builder.pin_memory)?;
3296    let pin_status = checkpoint_pin_status(pin_status)?;
3297    Ok(crate::LoaderConfiguration {
3298        batch_size: identity.batch_size,
3299        drop_last: identity.drop_last,
3300        workers: builder.workers,
3301        prefetch_factor: builder.prefetch_factor,
3302        in_order: true,
3303        loader_seed: builder.loader_seed,
3304        rank: builder.rank,
3305        replicas,
3306        distributed_shuffle,
3307        distributed_seed,
3308        distributed_drop_last,
3309        sampler_kind: identity.sampler_kind.clone(),
3310        pin_request,
3311        pin_status,
3312    })
3313}
3314
3315fn checkpoint_pin_request(request: PinRequest) -> Result<CheckpointPinRequest> {
3316    Ok(match request {
3317        PinRequest::Disabled => CheckpointPinRequest::Disabled,
3318        PinRequest::Auto => CheckpointPinRequest::Auto,
3319        PinRequest::Explicit(Device::Cuda(index)) => CheckpointPinRequest::ExplicitCuda { index },
3320        PinRequest::Explicit(device) => {
3321            return Err(invalid_configuration(
3322                "pin_memory",
3323                format!("unsupported checkpoint pin device {device:?}"),
3324            ));
3325        }
3326    })
3327}
3328
3329fn checkpoint_pin_status(status: PinMemoryStatus) -> Result<CheckpointPinStatus> {
3330    Ok(match status {
3331        PinMemoryStatus::Disabled => CheckpointPinStatus::Disabled,
3332        PinMemoryStatus::DisabledNoAccelerator => CheckpointPinStatus::DisabledNoAccelerator,
3333        PinMemoryStatus::Enabled(Device::Cuda(index)) => CheckpointPinStatus::Cuda { index },
3334        PinMemoryStatus::Enabled(device) => {
3335            return Err(invalid_configuration(
3336                "pin_memory",
3337                format!("unsupported effective checkpoint pin device {device:?}"),
3338            ));
3339        }
3340    })
3341}
3342
3343fn validate_static_loader_state_envelope<D, S, T, C>(
3344    state: &LoaderState<D, S, T, C>,
3345    identity: &str,
3346    builder: &BuilderConfiguration,
3347    pin_status: PinMemoryStatus,
3348    automatic_batching: bool,
3349    expected_generation: u64,
3350) -> Result<()> {
3351    let expected_batch_size = automatic_batching.then_some(builder.batch_size);
3352    let expected_drop_last = automatic_batching && builder.drop_last;
3353    let expected_pin_request = checkpoint_pin_request(builder.pin_memory)?;
3354    let expected_pin_status = checkpoint_pin_status(pin_status)?;
3355
3356    require_checkpoint_equal(
3357        "schema_version",
3358        state.schema_version,
3359        LOADER_STATE_SCHEMA_VERSION,
3360    )?;
3361    require_checkpoint_equal(
3362        "dataset_identity",
3363        state.dataset_identity.as_str(),
3364        identity,
3365    )?;
3366    require_checkpoint_equal(
3367        "iterator_generation",
3368        state.iterator_generation,
3369        expected_generation,
3370    )?;
3371    require_checkpoint_equal(
3372        "rng_derivation_version",
3373        state.rng_derivation_version,
3374        TASK_RNG_DERIVATION_VERSION,
3375    )?;
3376    require_checkpoint_equal(
3377        "worker_seed_derivation_version",
3378        state.worker_seed_derivation_version,
3379        WORKER_SEED_DERIVATION_VERSION,
3380    )?;
3381    require_checkpoint_equal(
3382        "batch_size",
3383        state.configuration.batch_size,
3384        expected_batch_size,
3385    )?;
3386    require_checkpoint_equal(
3387        "drop_last",
3388        state.configuration.drop_last,
3389        expected_drop_last,
3390    )?;
3391    require_checkpoint_equal("workers", state.configuration.workers, builder.workers)?;
3392    require_checkpoint_equal(
3393        "prefetch_factor",
3394        state.configuration.prefetch_factor,
3395        builder.prefetch_factor,
3396    )?;
3397    require_checkpoint_equal("in_order", state.configuration.in_order, builder.ordered)?;
3398    require_checkpoint_equal(
3399        "loader_seed",
3400        state.configuration.loader_seed,
3401        builder.loader_seed,
3402    )?;
3403    require_checkpoint_equal("rank", state.configuration.rank, builder.rank)?;
3404    require_checkpoint_equal(
3405        "pin_request",
3406        &state.configuration.pin_request,
3407        &expected_pin_request,
3408    )?;
3409    require_checkpoint_equal(
3410        "pin_status",
3411        &state.configuration.pin_status,
3412        &expected_pin_status,
3413    )
3414}
3415
3416fn validate_plan_loader_state_envelope<D, S, T, C>(
3417    state: &LoaderState<D, S, T, C>,
3418    expected: &crate::LoaderConfiguration,
3419) -> Result<()> {
3420    require_checkpoint_equal("replicas", state.configuration.replicas, expected.replicas)?;
3421    require_checkpoint_equal(
3422        "distributed_shuffle",
3423        state.configuration.distributed_shuffle,
3424        expected.distributed_shuffle,
3425    )?;
3426    require_checkpoint_equal(
3427        "distributed_seed",
3428        state.configuration.distributed_seed,
3429        expected.distributed_seed,
3430    )?;
3431    require_checkpoint_equal(
3432        "distributed_drop_last",
3433        state.configuration.distributed_drop_last,
3434        expected.distributed_drop_last,
3435    )?;
3436    require_checkpoint_equal(
3437        "sampler_kind",
3438        state.configuration.sampler_kind.as_str(),
3439        expected.sampler_kind.as_str(),
3440    )
3441}
3442
3443fn require_checkpoint_equal<T>(field: &'static str, actual: T, expected: T) -> Result<()>
3444where
3445    T: PartialEq,
3446{
3447    if actual != expected {
3448        return Err(invalid_configuration(
3449            field,
3450            "checkpoint value does not match the loader",
3451        ));
3452    }
3453    Ok(())
3454}
3455
3456fn resolve_pin_request(request: PinRequest) -> Result<PinMemoryStatus> {
3457    match request {
3458        PinRequest::Disabled => Ok(PinMemoryStatus::Disabled),
3459        PinRequest::Auto => {
3460            let capabilities = available_devices();
3461            if capabilities.cuda && capabilities.cuda_device_count > 0 {
3462                Ok(PinMemoryStatus::Enabled(Device::Cuda(0)))
3463            } else {
3464                Ok(PinMemoryStatus::DisabledNoAccelerator)
3465            }
3466        }
3467        PinRequest::Explicit(Device::Cuda(index)) => {
3468            let capabilities = available_devices();
3469            if capabilities.cuda && index < capabilities.cuda_device_count {
3470                Ok(PinMemoryStatus::Enabled(Device::Cuda(index)))
3471            } else {
3472                Err(invalid_configuration(
3473                    "pin_memory",
3474                    format!(
3475                        "CUDA device {index} was requested, but the linked runtime exposes {} available CUDA device(s)",
3476                        capabilities.cuda_device_count
3477                    ),
3478                ))
3479            }
3480        }
3481        PinRequest::Explicit(device) => Err(invalid_configuration(
3482            "pin_memory",
3483            format!("only an available CUDA device can back pinned host memory, got {device:?}"),
3484        )),
3485    }
3486}
3487
3488fn validate_auto_batch_boundary(
3489    exact_len: Option<usize>,
3490    batch_size: NonZeroUsize,
3491    drop_last: bool,
3492    next_batch: u64,
3493    next_logical_sample: u64,
3494) -> Result<()> {
3495    let batch_size = u64::try_from(batch_size.get()).map_err(|_| {
3496        invalid_configuration(
3497            "checkpoint",
3498            "batch size does not fit the checkpoint schema",
3499        )
3500    })?;
3501    let complete = next_logical_sample / batch_size;
3502    let aligned = next_logical_sample.is_multiple_of(batch_size);
3503    let final_short = match exact_len {
3504        Some(length) => {
3505            let length = u64::try_from(length).map_err(|_| {
3506                invalid_configuration(
3507                    "checkpoint",
3508                    "sampler length does not fit the checkpoint schema",
3509                )
3510            })?;
3511            !drop_last
3512                && next_logical_sample == length
3513                && !aligned
3514                && next_batch == complete.saturating_add(1)
3515        }
3516        None => false,
3517    };
3518    if (!aligned || next_batch != complete) && !final_short {
3519        return Err(invalid_configuration(
3520            "checkpoint",
3521            "batch and logical sample cursors are not on a visible batch boundary",
3522        ));
3523    }
3524    if let Some(length) = exact_len {
3525        let length = u64::try_from(length).map_err(|_| {
3526            invalid_configuration(
3527                "checkpoint",
3528                "sampler length does not fit the checkpoint schema",
3529            )
3530        })?;
3531        if next_logical_sample > length {
3532            return Err(invalid_configuration(
3533                "checkpoint",
3534                "logical sample cursor exceeds the sampler length",
3535            ));
3536        }
3537    }
3538    Ok(())
3539}
3540
3541fn batch_count(length: usize, batch_size: NonZeroUsize, drop_last: bool) -> usize {
3542    let batch_size = batch_size.get();
3543    let complete = length / batch_size;
3544    complete + usize::from(!drop_last && !length.is_multiple_of(batch_size))
3545}
3546
3547fn invalid_configuration(field: &'static str, reason: impl Into<String>) -> RustTorchError {
3548    RustTorchError::InvalidConfiguration {
3549        field,
3550        reason: reason.into(),
3551    }
3552}