Skip to main content

rusttorch_data/
loader.rs

1use std::{
2    collections::BTreeMap,
3    convert::Infallible,
4    marker::PhantomData,
5    num::NonZeroUsize,
6    panic::{AssertUnwindSafe, catch_unwind},
7    sync::Arc,
8    time::Duration,
9};
10
11use rusttorch_core::{Device, Result, RustTorchError, available_devices};
12
13#[path = "loader_checkpoint.rs"]
14mod checkpoint_workers;
15
16use crate::checkpoint::{StatelessTransformFactory, TransactionalTransformFactory};
17use crate::checkpoint::{
18    WorkerCheckpointActive, WorkerCheckpointIteration, WorkerLaneState, WorkerTransformLanes,
19    WorkerTransformState,
20};
21use crate::memory::{ByteBudget, MemoryDisabled, MemoryEnabled, MemoryPolicy};
22use crate::sampler::{
23    BatchSourceCheckpoint, DistributedConfiguration, SamplerCheckpoint, validate_batch_size,
24};
25use crate::worker::{
26    WorkerBatch, WorkerFailure, WorkerMessage, WorkerPool, WorkerPoolConfiguration, WorkerReceive,
27    WorkerRunContext, WorkerSubmit, WorkerTask, validate_worker_pool_capacity,
28};
29use crate::{
30    Auto, BatchSampler, BatchSource, CheckpointActive, CheckpointBuildError, CheckpointDisabled,
31    CheckpointFresh, CheckpointIteration, CheckpointPinRequest, CheckpointPinStatus,
32    CheckpointResume, Checkpointable, CloneTransformFactory, Collate, Dataset, DatasetCheckpoint,
33    Deadline, DefaultCollator, DefaultConverter, Explicit, IdentityTransformFactory,
34    LOADER_STATE_SCHEMA_VERSION, LoaderError, LoaderState, MemoryFootprint, NoWorkerInit,
35    PinDisabled, PinEnabled, PinMemory, PinMemoryStatus, PipelineError, RandomSampler, Sampler,
36    SequentialSampler, TASK_RNG_DERIVATION_VERSION, TaskContext, Transform, TransformFactory,
37    WORKER_SEED_DERIVATION_VERSION, WorkerCheckpoint, WorkerInit,
38};
39
40/// Type-level marker used only to make [`crate::DataLoader::builder`] inferable.
41#[doc(hidden)]
42pub struct BuilderDatasetMarker {
43    _private: (),
44}
45
46/// Default execution capability preserving fully local serial data types.
47#[derive(Clone, Copy, Debug, Default)]
48pub struct SerialExecution;
49
50/// Execution capability selected by [`DataLoaderBuilder::workers`].
51#[derive(Clone, Copy, Debug, Default)]
52pub struct WorkerExecution;
53
54trait SerialBoundaryPolicy {
55    fn begin_next(&mut self);
56    fn commit_visible(&mut self);
57}
58
59trait WorkerBoundaryPolicy<E> {
60    fn begin_next(&mut self) {}
61    fn submitted(&mut self, _sequence: u64, _logical_end: u64) {}
62    fn committed(&mut self, _sequence: u64, _serial_logical_end: u64) {}
63    fn defer_error(&mut self, _sequence: u64, error: E) -> Option<E> {
64        Some(error)
65    }
66    fn take_error(&mut self, _sequence: u64) -> Option<E> {
67        None
68    }
69}
70impl<E> WorkerBoundaryPolicy<E> for CheckpointDisabled {}
71impl<S, E> WorkerBoundaryPolicy<E> for WorkerCheckpointIteration<S, E> {
72    fn begin_next(&mut self) {
73        self.boundary_valid = false;
74    }
75    fn submitted(&mut self, sequence: u64, logical_end: u64) {
76        self.logical_ends.insert(sequence, logical_end);
77    }
78    fn committed(&mut self, sequence: u64, serial_logical_end: u64) {
79        self.next_logical_sample = self
80            .logical_ends
81            .remove(&sequence)
82            .unwrap_or(serial_logical_end);
83        self.boundary_valid = true;
84        if let Some(barrier) = &self.barrier {
85            barrier
86                .committed
87                .store(sequence + 1, std::sync::atomic::Ordering::Release);
88        }
89    }
90    fn defer_error(&mut self, sequence: u64, error: E) -> Option<E> {
91        self.errors.insert(sequence, error);
92        None
93    }
94    fn take_error(&mut self, sequence: u64) -> Option<E> {
95        self.errors.remove(&sequence)
96    }
97}
98
99impl SerialBoundaryPolicy for CheckpointDisabled {
100    fn begin_next(&mut self) {}
101    fn commit_visible(&mut self) {}
102}
103
104impl SerialBoundaryPolicy for CheckpointIteration {
105    fn begin_next(&mut self) {
106        self.boundary_valid = false;
107    }
108
109    fn commit_visible(&mut self) {
110        self.boundary_valid = true;
111    }
112}
113
114type TransformOutput<D, F> =
115    <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<
116        <D as Dataset>::Sample,
117    >>::Output;
118type TransformFailure<D, F> =
119    <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<
120        <D as Dataset>::Sample,
121    >>::Error;
122type PlanFailure<D, P, C, F> = <P as LoaderPlan<TransformOutput<D, F>, C>>::Error;
123type IterPipelineError<D, P, C, F, I> = PipelineError<
124    <D as Dataset>::Error,
125    TransformFailure<D, F>,
126    PlanFailure<D, P, C, F>,
127    <F as TransformFactory<<D as Dataset>::Sample>>::Error,
128    <I as WorkerInit>::Error,
129>;
130type IterError<D, P, C, F, I> = LoaderError<IterPipelineError<D, P, C, F, I>>;
131type IterResult<D, P, C, F, I> = std::result::Result<
132    <P as LoaderPlan<TransformOutput<D, F>, C>>::Batch,
133    IterError<D, P, C, F, I>,
134>;
135type CompletedBatch<D, F, M> = (
136    usize,
137    WorkerBatch<TransformOutput<D, F>, <M as MemoryPolicy>::Permit>,
138);
139type FootprintFn<D, F> = fn(&[TransformOutput<D, F>]) -> usize;
140type WorkerStageFailure<D, F, I> = WorkerFailure<
141    <D as Dataset>::Error,
142    TransformFailure<D, F>,
143    <F as TransformFactory<<D as Dataset>::Sample>>::Error,
144    <I as WorkerInit>::Error,
145>;
146
147#[doc(hidden)]
148pub trait MapPinPolicy<D, P, C, F>
149where
150    D: Dataset,
151    F: TransformFactory<D::Sample>,
152    P: LoaderPlan<TransformOutput<D, F>, C>,
153{
154    fn pin(batch: P::Batch, device: Device) -> Result<P::Batch>;
155}
156
157impl<D, P, C, F> MapPinPolicy<D, P, C, F> for PinDisabled
158where
159    D: Dataset,
160    F: TransformFactory<D::Sample>,
161    P: LoaderPlan<TransformOutput<D, F>, C>,
162{
163    fn pin(batch: P::Batch, _device: Device) -> Result<P::Batch> {
164        Ok(batch)
165    }
166}
167
168impl<D, P, C, F, Q> MapPinPolicy<D, P, C, F> for PinEnabled<Q>
169where
170    D: Dataset,
171    F: TransformFactory<D::Sample>,
172    P: LoaderPlan<TransformOutput<D, F>, C>,
173    P::Batch: PinMemory,
174{
175    fn pin(batch: P::Batch, device: Device) -> Result<P::Batch> {
176        batch.pin_memory(device)
177    }
178}
179
180struct PendingSubmission {
181    worker: usize,
182    task: WorkerTask,
183    next_submission: u64,
184    logical_end: u64,
185}
186
187impl Dataset for BuilderDatasetMarker {
188    type Sample = ();
189    type Error = Infallible;
190
191    fn len(&self) -> usize {
192        0
193    }
194
195    fn get(&self, _index: usize) -> std::result::Result<Self::Sample, Self::Error> {
196        unreachable!("the DataLoader builder marker never loads samples")
197    }
198}
199
200/// Automatic fixed-size batching over a reusable sampler.
201pub struct AutoBatch<S> {
202    sampler: S,
203    batch_size: NonZeroUsize,
204    drop_last: bool,
205}
206
207/// Explicit reusable batches of indices.
208pub struct ExplicitBatches<B> {
209    batches: B,
210}
211
212/// One-sample loading without an automatic batch dimension.
213pub struct NoBatch<S, V = DefaultConverter> {
214    sampler: S,
215    converter: V,
216}
217
218/// Internal plan behavior used by the named owned-loader iterator.
219#[doc(hidden)]
220pub trait LoaderPlan<Sample, C> {
221    /// Fresh iterator of index groups.
222    type Iter: Iterator<Item = Vec<usize>>;
223    /// Item yielded by the loader.
224    type Batch;
225    /// Typed dataset/collation pipeline error.
226    type Error;
227
228    /// Panic stage used while reading the plan epoch.
229    const EPOCH_PANIC_STAGE: &'static str = "sampler epoch";
230    /// Panic stage used while creating a fresh plan iterator.
231    const CREATION_PANIC_STAGE: &'static str = "sampler creation";
232    /// Panic stage used while refilling from the plan iterator.
233    const REFILL_PANIC_STAGE: &'static str = "sampler refill";
234    /// Panic stage used while producing the visible item.
235    const FINISH_PANIC_STAGE: &'static str = "collation";
236
237    /// Creates fresh index groups for the current epoch.
238    fn iter(&self) -> Self::Iter;
239    /// Returns the exact output count when known.
240    fn exact_len(&self) -> Option<usize>;
241    /// Returns the configured epoch.
242    fn epoch(&self) -> u64;
243    /// Forwards a new epoch to the reusable source.
244    fn set_epoch(&mut self, epoch: u64);
245    /// Converts fetched samples into one visible item.
246    fn finish(
247        &mut self,
248        collator: &mut C,
249        samples: Vec<Sample>,
250    ) -> std::result::Result<Self::Batch, Self::Error>;
251}
252
253/// Configuration-only behavior shared by loader index plans.
254#[doc(hidden)]
255pub trait LoaderPlanConfiguration {
256    /// Applies builder-level automatic batch controls when relevant.
257    fn apply_batch_options(&mut self, batch_size: NonZeroUsize, drop_last: bool);
258    /// Applies the initial epoch selected by the builder.
259    fn apply_epoch(&mut self, epoch: u64);
260}
261
262#[doc(hidden)]
263#[derive(Clone)]
264pub struct PlanCheckpointIdentity {
265    batch_size: Option<usize>,
266    drop_last: bool,
267    sampler_kind: String,
268    distributed: Option<DistributedConfiguration>,
269}
270
271#[doc(hidden)]
272pub trait CheckpointPlan<Sample, C>: LoaderPlan<Sample, C> {
273    type State;
274    type CoordinatorState;
275
276    const AUTOMATIC_BATCHING: bool = false;
277
278    fn checkpoint_identity(&self) -> PlanCheckpointIdentity;
279    /// Derives resume identity without observing a live plan or component.
280    fn checkpoint_identity_from_state(
281        state: &Self::State,
282        batch_size: NonZeroUsize,
283        drop_last: bool,
284    ) -> Result<PlanCheckpointIdentity>;
285    fn checkpoint_state(&self, next_batch: u64, next_logical_sample: u64) -> Result<Self::State>;
286    fn validate_checkpoint_state(
287        &self,
288        state: &Self::State,
289        epoch: u64,
290        next_batch: u64,
291        next_logical_sample: u64,
292    ) -> Result<()>;
293    fn validate_checkpoint_state_with_batch_options(
294        &self,
295        state: &Self::State,
296        epoch: u64,
297        next_batch: u64,
298        next_logical_sample: u64,
299        _batch_size: NonZeroUsize,
300        _drop_last: bool,
301    ) -> Result<()> {
302        self.validate_checkpoint_state(state, epoch, next_batch, next_logical_sample)
303    }
304    fn restore_checkpoint_iter_validated(&mut self, state: &Self::State) -> Self::Iter;
305    fn save_coordinator(&self, collator: &C) -> Self::CoordinatorState;
306    fn validate_coordinator(&self, collator: &C, state: &Self::CoordinatorState) -> Result<()>;
307    fn restore_coordinator_validated(&mut self, collator: &mut C, state: &Self::CoordinatorState);
308}
309
310impl<Sample, S, C> LoaderPlan<Sample, C> for AutoBatch<S>
311where
312    S: Sampler,
313    C: Collate<Sample>,
314{
315    type Iter = BatchSampler<S::Iter>;
316    type Batch = C::Batch;
317    type Error = C::Error;
318
319    const EPOCH_PANIC_STAGE: &'static str = "sampler epoch";
320    const CREATION_PANIC_STAGE: &'static str = "sampler creation";
321    const REFILL_PANIC_STAGE: &'static str = "sampler refill";
322    const FINISH_PANIC_STAGE: &'static str = "collation";
323
324    fn iter(&self) -> Self::Iter {
325        BatchSampler::from_validated(self.sampler.iter(), self.batch_size, self.drop_last)
326    }
327
328    fn exact_len(&self) -> Option<usize> {
329        self.sampler
330            .exact_len()
331            .map(|length| batch_count(length, self.batch_size, self.drop_last))
332    }
333
334    fn epoch(&self) -> u64 {
335        self.sampler.epoch()
336    }
337
338    fn set_epoch(&mut self, epoch: u64) {
339        self.sampler.set_epoch(epoch);
340    }
341
342    fn finish(
343        &mut self,
344        collator: &mut C,
345        samples: Vec<Sample>,
346    ) -> std::result::Result<Self::Batch, Self::Error> {
347        collator.collate(samples)
348    }
349}
350
351impl<S> LoaderPlanConfiguration for AutoBatch<S>
352where
353    S: Sampler,
354{
355    fn apply_batch_options(&mut self, batch_size: NonZeroUsize, drop_last: bool) {
356        self.batch_size = batch_size;
357        self.drop_last = drop_last;
358    }
359
360    fn apply_epoch(&mut self, epoch: u64) {
361        self.sampler.set_epoch(epoch);
362    }
363}
364
365impl<Sample, S, C> CheckpointPlan<Sample, C> for AutoBatch<S>
366where
367    S: SamplerCheckpoint,
368    C: Collate<Sample> + Checkpointable,
369{
370    type State = S::State;
371    type CoordinatorState = C::State;
372
373    const AUTOMATIC_BATCHING: bool = true;
374
375    fn checkpoint_identity(&self) -> PlanCheckpointIdentity {
376        PlanCheckpointIdentity {
377            batch_size: Some(self.batch_size.get()),
378            drop_last: self.drop_last,
379            sampler_kind: self.sampler.kind().to_owned(),
380            distributed: self.sampler.distributed_configuration(),
381        }
382    }
383
384    fn checkpoint_identity_from_state(
385        state: &Self::State,
386        batch_size: NonZeroUsize,
387        drop_last: bool,
388    ) -> Result<PlanCheckpointIdentity> {
389        Ok(PlanCheckpointIdentity {
390            batch_size: Some(batch_size.get()),
391            drop_last,
392            sampler_kind: S::checkpoint_kind_from_state(state),
393            distributed: S::checkpoint_distributed_configuration_from_state(state)?,
394        })
395    }
396
397    fn checkpoint_state(&self, next_batch: u64, next_logical_sample: u64) -> Result<Self::State> {
398        validate_auto_batch_boundary(
399            self.sampler.exact_len(),
400            self.batch_size,
401            self.drop_last,
402            next_batch,
403            next_logical_sample,
404        )?;
405        self.sampler.checkpoint_state(next_logical_sample)
406    }
407
408    fn validate_checkpoint_state(
409        &self,
410        state: &Self::State,
411        epoch: u64,
412        next_batch: u64,
413        next_logical_sample: u64,
414    ) -> Result<()> {
415        <Self as CheckpointPlan<Sample, C>>::validate_checkpoint_state_with_batch_options(
416            self,
417            state,
418            epoch,
419            next_batch,
420            next_logical_sample,
421            self.batch_size,
422            self.drop_last,
423        )
424    }
425
426    fn validate_checkpoint_state_with_batch_options(
427        &self,
428        state: &Self::State,
429        epoch: u64,
430        next_batch: u64,
431        next_logical_sample: u64,
432        batch_size: NonZeroUsize,
433        drop_last: bool,
434    ) -> Result<()> {
435        validate_auto_batch_boundary(
436            self.sampler.exact_len(),
437            batch_size,
438            drop_last,
439            next_batch,
440            next_logical_sample,
441        )?;
442        self.sampler
443            .validate_checkpoint_state(state, epoch, next_logical_sample)
444    }
445
446    fn restore_checkpoint_iter_validated(&mut self, state: &Self::State) -> Self::Iter {
447        let iterator = self.sampler.restore_checkpoint_iter_validated(state);
448        BatchSampler::from_validated(iterator, self.batch_size, self.drop_last)
449    }
450
451    fn save_coordinator(&self, collator: &C) -> Self::CoordinatorState {
452        collator.save_state()
453    }
454
455    fn validate_coordinator(&self, collator: &C, state: &Self::CoordinatorState) -> Result<()> {
456        collator.validate_state(state)
457    }
458
459    fn restore_coordinator_validated(&mut self, collator: &mut C, state: &Self::CoordinatorState) {
460        collator.load_validated(state);
461    }
462}
463
464impl<Sample, B, C> LoaderPlan<Sample, C> for ExplicitBatches<B>
465where
466    B: BatchSource,
467    C: Collate<Sample>,
468{
469    type Iter = B::Iter;
470    type Batch = C::Batch;
471    type Error = C::Error;
472
473    const EPOCH_PANIC_STAGE: &'static str = "batch source epoch";
474    const CREATION_PANIC_STAGE: &'static str = "batch source creation";
475    const REFILL_PANIC_STAGE: &'static str = "batch source refill";
476    const FINISH_PANIC_STAGE: &'static str = "collation";
477
478    fn iter(&self) -> Self::Iter {
479        self.batches.iter()
480    }
481
482    fn exact_len(&self) -> Option<usize> {
483        self.batches.exact_len()
484    }
485
486    fn epoch(&self) -> u64 {
487        self.batches.epoch()
488    }
489
490    fn set_epoch(&mut self, epoch: u64) {
491        self.batches.set_epoch(epoch);
492    }
493
494    fn finish(
495        &mut self,
496        collator: &mut C,
497        samples: Vec<Sample>,
498    ) -> std::result::Result<Self::Batch, Self::Error> {
499        collator.collate(samples)
500    }
501}
502
503impl<B> LoaderPlanConfiguration for ExplicitBatches<B>
504where
505    B: BatchSource,
506{
507    fn apply_batch_options(&mut self, _batch_size: NonZeroUsize, _drop_last: bool) {}
508
509    fn apply_epoch(&mut self, epoch: u64) {
510        self.batches.set_epoch(epoch);
511    }
512}
513
514impl<Sample, B, C> CheckpointPlan<Sample, C> for ExplicitBatches<B>
515where
516    B: BatchSourceCheckpoint,
517    C: Collate<Sample> + Checkpointable,
518{
519    type State = B::State;
520    type CoordinatorState = C::State;
521
522    fn checkpoint_identity(&self) -> PlanCheckpointIdentity {
523        PlanCheckpointIdentity {
524            batch_size: None,
525            drop_last: false,
526            sampler_kind: self.batches.kind(),
527            distributed: self.batches.distributed_configuration(),
528        }
529    }
530
531    fn checkpoint_identity_from_state(
532        state: &Self::State,
533        _batch_size: NonZeroUsize,
534        _drop_last: bool,
535    ) -> Result<PlanCheckpointIdentity> {
536        Ok(PlanCheckpointIdentity {
537            batch_size: None,
538            drop_last: false,
539            sampler_kind: B::checkpoint_kind_from_state(state),
540            distributed: B::checkpoint_distributed_configuration_from_state(state)?,
541        })
542    }
543
544    fn checkpoint_state(&self, next_batch: u64, next_logical_sample: u64) -> Result<Self::State> {
545        self.batches
546            .checkpoint_state(next_batch, next_logical_sample)
547    }
548
549    fn validate_checkpoint_state(
550        &self,
551        state: &Self::State,
552        epoch: u64,
553        next_batch: u64,
554        next_logical_sample: u64,
555    ) -> Result<()> {
556        self.batches
557            .validate_checkpoint_state(state, epoch, next_batch, next_logical_sample)
558    }
559
560    fn restore_checkpoint_iter_validated(&mut self, state: &Self::State) -> Self::Iter {
561        self.batches.restore_checkpoint_iter_validated(state)
562    }
563
564    fn save_coordinator(&self, collator: &C) -> Self::CoordinatorState {
565        collator.save_state()
566    }
567
568    fn validate_coordinator(&self, collator: &C, state: &Self::CoordinatorState) -> Result<()> {
569        collator.validate_state(state)
570    }
571
572    fn restore_coordinator_validated(&mut self, collator: &mut C, state: &Self::CoordinatorState) {
573        collator.load_validated(state);
574    }
575}
576
577fn singleton(index: usize) -> Vec<usize> {
578    vec![index]
579}
580
581impl<Sample, S, V, C> LoaderPlan<Sample, C> for NoBatch<S, V>
582where
583    S: Sampler,
584    V: Collate<Sample>,
585{
586    type Iter = std::iter::Map<S::Iter, fn(usize) -> Vec<usize>>;
587    type Batch = V::Batch;
588    type Error = V::Error;
589
590    const EPOCH_PANIC_STAGE: &'static str = "sampler epoch";
591    const CREATION_PANIC_STAGE: &'static str = "sampler creation";
592    const REFILL_PANIC_STAGE: &'static str = "sampler refill";
593    const FINISH_PANIC_STAGE: &'static str = "conversion";
594
595    fn iter(&self) -> Self::Iter {
596        self.sampler.iter().map(singleton)
597    }
598
599    fn exact_len(&self) -> Option<usize> {
600        self.sampler.exact_len()
601    }
602
603    fn epoch(&self) -> u64 {
604        self.sampler.epoch()
605    }
606
607    fn set_epoch(&mut self, epoch: u64) {
608        self.sampler.set_epoch(epoch);
609    }
610
611    fn finish(
612        &mut self,
613        _collator: &mut C,
614        samples: Vec<Sample>,
615    ) -> std::result::Result<Self::Batch, Self::Error> {
616        self.converter.collate(samples)
617    }
618}
619
620impl<S, V> LoaderPlanConfiguration for NoBatch<S, V>
621where
622    S: Sampler,
623{
624    fn apply_batch_options(&mut self, _batch_size: NonZeroUsize, _drop_last: bool) {}
625
626    fn apply_epoch(&mut self, epoch: u64) {
627        self.sampler.set_epoch(epoch);
628    }
629}
630
631impl<Sample, S, V, C> CheckpointPlan<Sample, C> for NoBatch<S, V>
632where
633    S: SamplerCheckpoint,
634    V: Collate<Sample> + Checkpointable,
635{
636    type State = S::State;
637    type CoordinatorState = V::State;
638
639    fn checkpoint_identity(&self) -> PlanCheckpointIdentity {
640        PlanCheckpointIdentity {
641            batch_size: None,
642            drop_last: false,
643            sampler_kind: self.sampler.kind().to_owned(),
644            distributed: self.sampler.distributed_configuration(),
645        }
646    }
647
648    fn checkpoint_identity_from_state(
649        state: &Self::State,
650        _batch_size: NonZeroUsize,
651        _drop_last: bool,
652    ) -> Result<PlanCheckpointIdentity> {
653        Ok(PlanCheckpointIdentity {
654            batch_size: None,
655            drop_last: false,
656            sampler_kind: S::checkpoint_kind_from_state(state),
657            distributed: S::checkpoint_distributed_configuration_from_state(state)?,
658        })
659    }
660
661    fn checkpoint_state(&self, next_batch: u64, next_logical_sample: u64) -> Result<Self::State> {
662        if next_batch != next_logical_sample {
663            return Err(invalid_configuration(
664                "checkpoint",
665                "no-batch cursor must advance one batch per logical sample",
666            ));
667        }
668        self.sampler.checkpoint_state(next_logical_sample)
669    }
670
671    fn validate_checkpoint_state(
672        &self,
673        state: &Self::State,
674        epoch: u64,
675        next_batch: u64,
676        next_logical_sample: u64,
677    ) -> Result<()> {
678        if next_batch != next_logical_sample {
679            return Err(invalid_configuration(
680                "checkpoint",
681                "no-batch cursor must advance one batch per logical sample",
682            ));
683        }
684        self.sampler
685            .validate_checkpoint_state(state, epoch, next_logical_sample)
686    }
687
688    fn restore_checkpoint_iter_validated(&mut self, state: &Self::State) -> Self::Iter {
689        self.sampler
690            .restore_checkpoint_iter_validated(state)
691            .map(singleton as fn(usize) -> Vec<usize>)
692    }
693
694    fn save_coordinator(&self, _collator: &C) -> Self::CoordinatorState {
695        self.converter.save_state()
696    }
697
698    fn validate_coordinator(&self, _collator: &C, state: &Self::CoordinatorState) -> Result<()> {
699        self.converter.validate_state(state)
700    }
701
702    fn restore_coordinator_validated(&mut self, _collator: &mut C, state: &Self::CoordinatorState) {
703        self.converter.load_validated(state);
704    }
705}
706
707#[derive(Clone, Copy, Default)]
708struct ExplicitArguments {
709    batch_size: bool,
710    drop_last: bool,
711    sampler: bool,
712    shuffle: bool,
713    batch_sampler: bool,
714    without_batching: bool,
715    prefetch_factor: bool,
716    prefetch_bytes: bool,
717}
718
719#[derive(Clone, Copy)]
720enum PinRequest {
721    Disabled,
722    Auto,
723    Explicit(Device),
724}
725
726#[derive(Clone, Copy)]
727struct BuilderConfiguration {
728    batch_size: usize,
729    drop_last: bool,
730    workers: usize,
731    persistent_workers: bool,
732    timeout: Option<Duration>,
733    ordered: bool,
734    pin_memory: PinRequest,
735    prefetch_factor: Option<usize>,
736    prefetch_bytes: Option<NonZeroUsize>,
737    loader_seed: u64,
738    epoch: u64,
739    rank: usize,
740}
741
742impl Default for BuilderConfiguration {
743    fn default() -> Self {
744        Self {
745            batch_size: 1,
746            drop_last: false,
747            workers: 0,
748            persistent_workers: false,
749            timeout: None,
750            ordered: true,
751            pin_memory: PinRequest::Disabled,
752            prefetch_factor: None,
753            prefetch_bytes: None,
754            loader_seed: 0,
755            epoch: 0,
756            rank: 0,
757        }
758    }
759}
760
761/// Builder for an owned, re-iterable map-style data loader.
762pub struct DataLoaderBuilder<
763    D,
764    P,
765    C,
766    F = IdentityTransformFactory,
767    I = NoWorkerInit,
768    X = SerialExecution,
769    M = MemoryDisabled,
770    N = PinDisabled,
771    K = CheckpointDisabled,
772> {
773    dataset: D,
774    plan: P,
775    collator: C,
776    transform_factory: F,
777    worker_init: I,
778    configuration: BuilderConfiguration,
779    explicit: ExplicitArguments,
780    checkpoint: K,
781    states: PhantomData<(X, M, N)>,
782}
783
784impl<D> DataLoaderBuilder<D, AutoBatch<SequentialSampler>, DefaultCollator>
785where
786    D: Dataset,
787{
788    /// Creates the default serial builder for `dataset`.
789    pub fn new(dataset: D) -> Self {
790        Self {
791            plan: AutoBatch {
792                sampler: SequentialSampler::new(dataset.len()),
793                batch_size: NonZeroUsize::MIN,
794                drop_last: false,
795            },
796            dataset,
797            collator: DefaultCollator,
798            transform_factory: IdentityTransformFactory,
799            worker_init: NoWorkerInit,
800            configuration: BuilderConfiguration::default(),
801            explicit: ExplicitArguments::default(),
802            checkpoint: CheckpointDisabled,
803            states: PhantomData,
804        }
805    }
806}
807
808impl<D, P, C, F, I, X, M, N, K> DataLoaderBuilder<D, P, C, F, I, X, M, N, K> {
809    fn map<P2, C2>(
810        self,
811        transform: impl FnOnce(P, C) -> (P2, C2),
812    ) -> DataLoaderBuilder<D, P2, C2, F, I, X, M, N, K> {
813        let (plan, collator) = transform(self.plan, self.collator);
814        DataLoaderBuilder {
815            dataset: self.dataset,
816            plan,
817            collator,
818            transform_factory: self.transform_factory,
819            worker_init: self.worker_init,
820            configuration: self.configuration,
821            explicit: self.explicit,
822            checkpoint: self.checkpoint,
823            states: PhantomData,
824        }
825    }
826
827    /// Sets the automatic batch size.
828    pub fn batch_size(mut self, batch_size: usize) -> Self {
829        self.configuration.batch_size = batch_size;
830        self.explicit.batch_size = true;
831        self
832    }
833
834    /// Selects whether a short final automatic batch is omitted.
835    pub fn drop_last(mut self, drop_last: bool) -> Self {
836        self.configuration.drop_last = drop_last;
837        self.explicit.drop_last = true;
838        self
839    }
840
841    /// Selects bounded map-worker execution and its thread-safe capability.
842    ///
843    /// Positive counts share the map dataset through [`Arc`], fetch and
844    /// transform in deterministic worker lanes, and collate on the calling
845    /// thread. Passing zero keeps serial execution, but selecting this method
846    /// still requires worker-safe types at compile time.
847    pub fn workers(
848        mut self,
849        workers: usize,
850    ) -> DataLoaderBuilder<D, P, C, F, I, WorkerExecution, M, N, K> {
851        self.configuration.workers = workers;
852        DataLoaderBuilder {
853            dataset: self.dataset,
854            plan: self.plan,
855            collator: self.collator,
856            transform_factory: self.transform_factory,
857            worker_init: self.worker_init,
858            configuration: self.configuration,
859            explicit: self.explicit,
860            checkpoint: self.checkpoint,
861            states: PhantomData,
862        }
863    }
864
865    /// Sets the seed used by deterministic task and worker contexts.
866    pub fn seed(mut self, seed: u64) -> Self {
867        self.configuration.loader_seed = seed;
868        self
869    }
870
871    /// Sets the initial epoch used by sampling and task contexts.
872    pub fn epoch(mut self, epoch: u64) -> Self {
873        self.configuration.epoch = epoch;
874        self
875    }
876
877    /// Sets the distributed rank included in deterministic contexts.
878    pub fn rank(mut self, rank: usize) -> Self {
879        self.configuration.rank = rank;
880        self
881    }
882
883    /// Clones `transform` once for each serial iterator or worker lifecycle.
884    pub fn transform<T>(
885        self,
886        transform: T,
887    ) -> DataLoaderBuilder<D, P, C, CloneTransformFactory<T>, I, X, M, N, K> {
888        DataLoaderBuilder {
889            dataset: self.dataset,
890            plan: self.plan,
891            collator: self.collator,
892            transform_factory: CloneTransformFactory::new(transform),
893            worker_init: self.worker_init,
894            configuration: self.configuration,
895            explicit: self.explicit,
896            checkpoint: self.checkpoint,
897            states: PhantomData,
898        }
899    }
900
901    /// Replaces transform construction with an explicit typed factory.
902    pub fn transform_factory<F2>(
903        self,
904        factory: F2,
905    ) -> DataLoaderBuilder<D, P, C, F2, I, X, M, N, K> {
906        DataLoaderBuilder {
907            dataset: self.dataset,
908            plan: self.plan,
909            collator: self.collator,
910            transform_factory: factory,
911            worker_init: self.worker_init,
912            configuration: self.configuration,
913            explicit: self.explicit,
914            checkpoint: self.checkpoint,
915            states: PhantomData,
916        }
917    }
918
919    /// Marks the current factory's transform as explicitly stateless for
920    /// exact checkpointing.
921    pub fn checkpoint_stateless(
922        self,
923    ) -> DataLoaderBuilder<D, P, C, StatelessTransformFactory<F>, I, X, M, N, K> {
924        DataLoaderBuilder {
925            dataset: self.dataset,
926            plan: self.plan,
927            collator: self.collator,
928            transform_factory: StatelessTransformFactory(self.transform_factory),
929            worker_init: self.worker_init,
930            configuration: self.configuration,
931            explicit: self.explicit,
932            checkpoint: self.checkpoint,
933            states: PhantomData,
934        }
935    }
936
937    /// Marks the current factory's transform as transactional for exact
938    /// checkpointing.
939    pub fn checkpoint_transactional(
940        self,
941    ) -> DataLoaderBuilder<D, P, C, TransactionalTransformFactory<F>, I, X, M, N, K> {
942        DataLoaderBuilder {
943            dataset: self.dataset,
944            plan: self.plan,
945            collator: self.collator,
946            transform_factory: TransactionalTransformFactory(self.transform_factory),
947            worker_init: self.worker_init,
948            configuration: self.configuration,
949            explicit: self.explicit,
950            checkpoint: self.checkpoint,
951            states: PhantomData,
952        }
953    }
954
955    /// Stores the initializer used by future positive-worker execution.
956    pub fn worker_init<I2>(self, worker_init: I2) -> DataLoaderBuilder<D, P, C, F, I2, X, M, N, K> {
957        DataLoaderBuilder {
958            dataset: self.dataset,
959            plan: self.plan,
960            collator: self.collator,
961            transform_factory: self.transform_factory,
962            worker_init,
963            configuration: self.configuration,
964            explicit: self.explicit,
965            checkpoint: self.checkpoint,
966            states: PhantomData,
967        }
968    }
969
970    /// Selects whether positive-worker iterators reuse loader-owned threads.
971    ///
972    /// Persistent workers keep their initial [`crate::WorkerInfo`] seeds and
973    /// factory-created state across epochs. Each iterator still receives a
974    /// fresh generation cancellation token and deadline, and dropping either
975    /// the iterator or loader cooperatively wakes and joins the affected work.
976    pub fn persistent_workers(mut self, persistent: bool) -> Self {
977        self.configuration.persistent_workers = persistent;
978        self
979    }
980
981    /// Sets the cooperative timeout for each blocking [`Iterator::next`] call.
982    ///
983    /// [`Duration::ZERO`] disables the timeout. Expiry cancels the current
984    /// generation, drains it to quiescence, yields one typed timeout error, and
985    /// then ends the iterator. User dataset and transform code must observe its
986    /// [`crate::WorkerContext`] or [`crate::TaskContext`] to stop promptly;
987    /// Rust threads are never force-cancelled. A nonzero timeout that exceeds
988    /// the platform monotonic clock range is rejected by [`Self::build`].
989    pub fn timeout(mut self, timeout: Duration) -> Self {
990        self.configuration.timeout = (!timeout.is_zero()).then_some(timeout);
991        self
992    }
993
994    /// Selects ordered or completion-order delivery.
995    pub fn ordered(mut self, ordered: bool) -> Self {
996        self.configuration.ordered = ordered;
997        self
998    }
999
1000    /// Selects sampler-order or completion-order delivery.
1001    pub fn in_order(self, in_order: bool) -> Self {
1002        self.ordered(in_order)
1003    }
1004
1005    /// Enables automatic recursive pinning for CUDA device zero when available.
1006    ///
1007    /// Enabling pinning requires the final batch type to implement
1008    /// [`PinMemory`] at build time:
1009    ///
1010    /// ```compile_fail
1011    /// use std::convert::Infallible;
1012    /// use rusttorch_data::{DataLoader, Dataset, FnCollate};
1013    ///
1014    /// struct Rows;
1015    /// struct Batch;
1016    /// impl Dataset for Rows {
1017    ///     type Sample = u8;
1018    ///     type Error = Infallible;
1019    ///     fn len(&self) -> usize { 1 }
1020    ///     fn get(&self, _: usize) -> Result<u8, Infallible> { Ok(1) }
1021    /// }
1022    ///
1023    /// let _ = DataLoader::builder(Rows)
1024    ///     .collate(FnCollate::new(|_: Vec<u8>| Ok::<_, Infallible>(Batch)))
1025    ///     .pin_memory()
1026    ///     .build();
1027    /// ```
1028    pub fn pin_memory(mut self) -> DataLoaderBuilder<D, P, C, F, I, X, M, PinEnabled<Auto>, K> {
1029        self.configuration.pin_memory = PinRequest::Auto;
1030        DataLoaderBuilder {
1031            dataset: self.dataset,
1032            plan: self.plan,
1033            collator: self.collator,
1034            transform_factory: self.transform_factory,
1035            worker_init: self.worker_init,
1036            configuration: self.configuration,
1037            explicit: self.explicit,
1038            checkpoint: self.checkpoint,
1039            states: PhantomData,
1040        }
1041    }
1042
1043    /// Enables recursive pinning for one explicit, available CUDA device.
1044    pub fn pin_memory_for(
1045        mut self,
1046        device: Device,
1047    ) -> DataLoaderBuilder<D, P, C, F, I, X, M, PinEnabled<Explicit>, K> {
1048        self.configuration.pin_memory = PinRequest::Explicit(device);
1049        DataLoaderBuilder {
1050            dataset: self.dataset,
1051            plan: self.plan,
1052            collator: self.collator,
1053            transform_factory: self.transform_factory,
1054            worker_init: self.worker_init,
1055            configuration: self.configuration,
1056            explicit: self.explicit,
1057            checkpoint: self.checkpoint,
1058            states: PhantomData,
1059        }
1060    }
1061
1062    /// Sets bounded batches prefetched per worker.
1063    ///
1064    /// The worker count multiplied by this factor is checked before any
1065    /// worker starts and is also the global outstanding-work credit limit.
1066    /// Capacities that overflow crossbeam's ring arithmetic or whose
1067    /// conservative aggregate of concrete task/control/completion slots, worker
1068    /// bookkeeping, and channel control-block allowances exceeds the checked
1069    /// 64 MiB queue-allocation ceiling are rejected during build.
1070    pub fn prefetch_factor(mut self, factor: usize) -> Self {
1071        self.configuration.prefetch_factor = Some(factor);
1072        self.explicit.prefetch_factor = true;
1073        self
1074    }
1075
1076    /// Enables a nonzero byte budget for final post-transform prefetched data.
1077    ///
1078    /// The coordinator's active collation batch is outside this queue budget
1079    /// and remains bounded by the configured batch size.
1080    ///
1081    /// Enabling the budget requires the final transformed type to implement
1082    /// [`MemoryFootprint`] at build time:
1083    ///
1084    /// ```compile_fail
1085    /// use std::{convert::Infallible, num::NonZeroUsize};
1086    /// use rusttorch_data::{DataLoader, Dataset, VecCollate};
1087    ///
1088    /// struct Sample;
1089    /// struct Rows;
1090    /// impl Dataset for Rows {
1091    ///     type Sample = Sample;
1092    ///     type Error = Infallible;
1093    ///     fn len(&self) -> usize { 1 }
1094    ///     fn get(&self, _: usize) -> Result<Sample, Infallible> { Ok(Sample) }
1095    /// }
1096    ///
1097    /// let _ = DataLoader::builder(Rows)
1098    ///     .workers(1)
1099    ///     .collate(VecCollate)
1100    ///     .prefetch_bytes(NonZeroUsize::MIN)
1101    ///     .build();
1102    /// ```
1103    pub fn prefetch_bytes(
1104        mut self,
1105        limit: NonZeroUsize,
1106    ) -> DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, N, K> {
1107        self.configuration.prefetch_bytes = Some(limit);
1108        self.explicit.prefetch_bytes = true;
1109        DataLoaderBuilder {
1110            dataset: self.dataset,
1111            plan: self.plan,
1112            collator: self.collator,
1113            transform_factory: self.transform_factory,
1114            worker_init: self.worker_init,
1115            configuration: self.configuration,
1116            explicit: self.explicit,
1117            checkpoint: self.checkpoint,
1118            states: PhantomData,
1119        }
1120    }
1121
1122    /// Replaces automatic batching with explicit reusable index batches.
1123    pub fn batch_sampler<B>(
1124        mut self,
1125        batches: B,
1126    ) -> DataLoaderBuilder<D, ExplicitBatches<B>, C, F, I, X, M, N, K> {
1127        self.explicit.batch_sampler = true;
1128        self.map(|_, collator| (ExplicitBatches { batches }, collator))
1129    }
1130
1131    /// Validates configuration and constructs the owned loader.
1132    ///
1133    /// # Errors
1134    ///
1135    /// Returns [`RustTorchError::InvalidConfiguration`] before plan callbacks
1136    /// for incompatible zero-worker options, zero-valued controls, or
1137    /// unallocatable worker queue sizes.
1138    #[allow(clippy::type_complexity)]
1139    fn build_inner(
1140        mut self,
1141        footprint: Option<FootprintFn<D, F>>,
1142    ) -> Result<OwnedDataLoader<D, P, C, F, I, X, M, N, K>>
1143    where
1144        D: Dataset,
1145        F: TransformFactory<D::Sample>,
1146        P: LoaderPlanConfiguration,
1147        I: WorkerInit,
1148        M: MemoryPolicy,
1149    {
1150        validate_configuration(&self.configuration, self.explicit)?;
1151        let pin_memory_status = resolve_pin_request(self.configuration.pin_memory)?;
1152        let batch_size = validate_batch_size(self.configuration.batch_size)?;
1153        let effective_prefetch = match (
1154            self.configuration.workers,
1155            self.configuration.prefetch_factor,
1156        ) {
1157            (0, _) => None,
1158            (_, Some(factor)) => NonZeroUsize::new(factor),
1159            (_, None) => NonZeroUsize::new(2),
1160        };
1161        let outstanding_capacity = match (self.configuration.workers, effective_prefetch) {
1162            (0, _) => None,
1163            (workers, Some(factor)) => {
1164                Some(workers.checked_mul(factor.get()).ok_or_else(|| {
1165                    invalid_configuration(
1166                        "prefetch_factor",
1167                        "workers multiplied by prefetch_factor exceeds usize",
1168                    )
1169                })?)
1170            }
1171            (_, None) => unreachable!("positive workers always have an effective prefetch factor"),
1172        };
1173        if let Some(capacity) = outstanding_capacity {
1174            validate_worker_pool_capacity::<D, F, I, M>(
1175                self.configuration.workers,
1176                effective_prefetch
1177                    .expect("positive workers have a prefetch factor")
1178                    .get(),
1179                capacity,
1180            )?;
1181        }
1182        self.plan
1183            .apply_batch_options(batch_size, self.configuration.drop_last);
1184        self.plan.apply_epoch(self.configuration.epoch);
1185        Ok(OwnedDataLoader {
1186            dataset: Arc::new(self.dataset),
1187            plan: self.plan,
1188            collator: self.collator,
1189            transform_factory: Arc::new(self.transform_factory),
1190            worker_init: Arc::new(self.worker_init),
1191            workers: self.configuration.workers,
1192            persistent_workers: self.configuration.persistent_workers,
1193            timeout: self.configuration.timeout,
1194            ordered: self.configuration.ordered,
1195            pin_memory_status,
1196            footprint,
1197            effective_prefetch_bytes: self.configuration.prefetch_bytes,
1198            effective_prefetch,
1199            outstanding_capacity,
1200            loader_seed: self.configuration.loader_seed,
1201            rank: self.configuration.rank,
1202            next_generation: 0,
1203            persistent_pool: None,
1204            checkpoint: self.checkpoint,
1205            policies: PhantomData,
1206        })
1207    }
1208}
1209
1210impl<D, P, C, F, I, X, M, N> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointDisabled> {
1211    /// Enables exact checkpointing with a caller-defined dataset identity.
1212    ///
1213    /// The identity must describe the exact dataset contents, not merely its
1214    /// Rust type or path. Empty identities are rejected by [`Self::build`].
1215    pub fn dataset_identity(
1216        self,
1217        identity: String,
1218    ) -> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointFresh> {
1219        DataLoaderBuilder {
1220            dataset: self.dataset,
1221            plan: self.plan,
1222            collator: self.collator,
1223            transform_factory: self.transform_factory,
1224            worker_init: self.worker_init,
1225            configuration: self.configuration,
1226            explicit: self.explicit,
1227            checkpoint: CheckpointFresh { identity },
1228            states: PhantomData,
1229        }
1230    }
1231}
1232
1233impl<D, P, C, F, I, X, M, N> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointFresh> {
1234    /// Selects a typed serial loader checkpoint for validated resume.
1235    pub fn resume_from<S>(
1236        self,
1237        state: S,
1238    ) -> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointResume<S>> {
1239        DataLoaderBuilder {
1240            dataset: self.dataset,
1241            plan: self.plan,
1242            collator: self.collator,
1243            transform_factory: self.transform_factory,
1244            worker_init: self.worker_init,
1245            configuration: self.configuration,
1246            explicit: self.explicit,
1247            checkpoint: CheckpointResume {
1248                identity: self.checkpoint.identity,
1249                state,
1250            },
1251            states: PhantomData,
1252        }
1253    }
1254}
1255
1256fn resident_bytes<T: MemoryFootprint>(values: &[T]) -> usize {
1257    let mut total = 0usize;
1258    for value in values {
1259        let Some(next) = total.checked_add(value.resident_bytes()) else {
1260            return usize::MAX;
1261        };
1262        total = next;
1263    }
1264    total
1265}
1266
1267impl<D, P, C, F, I, X>
1268    DataLoaderBuilder<D, P, C, F, I, X, MemoryDisabled, PinDisabled, CheckpointDisabled>
1269{
1270    /// Validates configuration and builds an item-bounded, unpinned loader.
1271    #[allow(clippy::type_complexity)]
1272    pub fn build(
1273        self,
1274    ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryDisabled, PinDisabled, CheckpointDisabled>>
1275    where
1276        D: Dataset,
1277        F: TransformFactory<D::Sample>,
1278        P: LoaderPlanConfiguration,
1279        I: WorkerInit,
1280    {
1281        self.build_inner(None)
1282    }
1283}
1284
1285impl<D, P, C, F, I, X>
1286    DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, PinDisabled, CheckpointDisabled>
1287{
1288    /// Validates configuration and builds a byte-bounded, unpinned loader.
1289    #[allow(clippy::type_complexity)]
1290    pub fn build(
1291        self,
1292    ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryEnabled, PinDisabled, CheckpointDisabled>>
1293    where
1294        D: Dataset,
1295        F: TransformFactory<D::Sample>,
1296        TransformOutput<D, F>: MemoryFootprint,
1297        P: LoaderPlanConfiguration,
1298        I: WorkerInit,
1299    {
1300        self.build_inner(Some(resident_bytes::<TransformOutput<D, F>>))
1301    }
1302}
1303
1304impl<D, P, C, F, I, X, Q>
1305    DataLoaderBuilder<D, P, C, F, I, X, MemoryDisabled, PinEnabled<Q>, CheckpointDisabled>
1306{
1307    /// Validates configuration and builds an item-bounded pinned loader.
1308    #[allow(clippy::type_complexity)]
1309    pub fn build(
1310        self,
1311    ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryDisabled, PinEnabled<Q>, CheckpointDisabled>>
1312    where
1313        D: Dataset,
1314        F: TransformFactory<D::Sample>,
1315        P: LoaderPlanConfiguration + LoaderPlan<TransformOutput<D, F>, C>,
1316        P::Batch: PinMemory,
1317        I: WorkerInit,
1318    {
1319        self.build_inner(None)
1320    }
1321}
1322
1323impl<D, P, C, F, I, X, Q>
1324    DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, PinEnabled<Q>, CheckpointDisabled>
1325{
1326    /// Validates configuration and builds a byte-bounded pinned loader.
1327    #[allow(clippy::type_complexity)]
1328    pub fn build(
1329        self,
1330    ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryEnabled, PinEnabled<Q>, CheckpointDisabled>>
1331    where
1332        D: Dataset,
1333        F: TransformFactory<D::Sample>,
1334        TransformOutput<D, F>: MemoryFootprint,
1335        P: LoaderPlanConfiguration + LoaderPlan<TransformOutput<D, F>, C>,
1336        P::Batch: PinMemory,
1337        I: WorkerInit,
1338    {
1339        self.build_inner(Some(resident_bytes::<TransformOutput<D, F>>))
1340    }
1341}
1342
1343#[allow(private_bounds)]
1344impl<D, P, C, F, I, N>
1345    DataLoaderBuilder<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointFresh>
1346where
1347    D: DatasetCheckpoint,
1348    F: TransformFactory<D::Sample>,
1349    F::Transform: WorkerCheckpoint,
1350    P: LoaderPlanConfiguration
1351        + LoaderPlan<TransformOutput<D, F>, C>
1352        + CheckpointPlan<TransformOutput<D, F>, C>,
1353    I: WorkerInit,
1354    N: MapPinPolicy<D, P, C, F>,
1355{
1356    /// Builds a fresh exact serial loader without constructing its transform.
1357    #[allow(clippy::type_complexity)]
1358    pub fn build(
1359        mut self,
1360    ) -> Result<
1361        OwnedDataLoader<
1362            D,
1363            P,
1364            C,
1365            F,
1366            I,
1367            SerialExecution,
1368            MemoryDisabled,
1369            N,
1370            CheckpointActive<P::Iter, F::Transform>,
1371        >,
1372    > {
1373        validate_exact_builder(
1374            &self.configuration,
1375            self.explicit,
1376            &self.checkpoint.identity,
1377        )?;
1378        let pin_memory_status = resolve_pin_request(self.configuration.pin_memory)?;
1379        let batch_size = validate_batch_size(self.configuration.batch_size)?;
1380        self.plan
1381            .apply_batch_options(batch_size, self.configuration.drop_last);
1382        self.plan.apply_epoch(self.configuration.epoch);
1383        let identity = self.plan.checkpoint_identity();
1384        validate_distributed_rank(&identity, self.configuration.rank)?;
1385        let checkpoint_configuration =
1386            checkpoint_configuration(&identity, &self.configuration, pin_memory_status)?;
1387
1388        Ok(OwnedDataLoader {
1389            dataset: Arc::new(self.dataset),
1390            plan: self.plan,
1391            collator: self.collator,
1392            transform_factory: Arc::new(self.transform_factory),
1393            worker_init: Arc::new(self.worker_init),
1394            workers: 0,
1395            persistent_workers: false,
1396            timeout: None,
1397            ordered: true,
1398            pin_memory_status,
1399            footprint: None,
1400            effective_prefetch_bytes: None,
1401            effective_prefetch: None,
1402            outstanding_capacity: None,
1403            loader_seed: self.configuration.loader_seed,
1404            rank: self.configuration.rank,
1405            next_generation: 0,
1406            persistent_pool: None,
1407            checkpoint: CheckpointActive {
1408                identity: self.checkpoint.identity,
1409                configuration: checkpoint_configuration,
1410                pending: None,
1411            },
1412            policies: PhantomData,
1413        })
1414    }
1415}
1416
1417#[allow(private_bounds)]
1418impl<D, P, C, F, I, N, DS, SS, TS, CS>
1419    DataLoaderBuilder<
1420        D,
1421        P,
1422        C,
1423        F,
1424        I,
1425        SerialExecution,
1426        MemoryDisabled,
1427        N,
1428        CheckpointResume<LoaderState<DS, SS, TS, CS>>,
1429    >
1430where
1431    D: DatasetCheckpoint<State = DS>,
1432    F: TransformFactory<D::Sample>,
1433    F::Transform: WorkerCheckpoint<State = TS>,
1434    P: LoaderPlanConfiguration
1435        + LoaderPlan<TransformOutput<D, F>, C>
1436        + CheckpointPlan<TransformOutput<D, F>, C, State = SS, CoordinatorState = CS>,
1437    I: WorkerInit,
1438    N: MapPinPolicy<D, P, C, F>,
1439{
1440    /// Validates and restores an exact serial loader transactionally.
1441    #[allow(clippy::type_complexity)]
1442    pub fn build(
1443        mut self,
1444    ) -> std::result::Result<
1445        OwnedDataLoader<
1446            D,
1447            P,
1448            C,
1449            F,
1450            I,
1451            SerialExecution,
1452            MemoryDisabled,
1453            N,
1454            CheckpointActive<P::Iter, F::Transform>,
1455        >,
1456        CheckpointBuildError<F::Error>,
1457    > {
1458        let CheckpointResume {
1459            identity: requested_identity,
1460            state,
1461        } = self.checkpoint;
1462        validate_exact_builder(&self.configuration, self.explicit, &requested_identity)?;
1463        let pin_memory_status = resolve_pin_request(self.configuration.pin_memory)?;
1464        let batch_size = validate_batch_size(self.configuration.batch_size)?;
1465        validate_static_loader_state_envelope(
1466            &state,
1467            &requested_identity,
1468            &self.configuration,
1469            pin_memory_status,
1470            <P as CheckpointPlan<TransformOutput<D, F>, C>>::AUTOMATIC_BATCHING,
1471            0,
1472        )?;
1473
1474        let plan_identity =
1475            <P as CheckpointPlan<TransformOutput<D, F>, C>>::checkpoint_identity_from_state(
1476                &state.sampler,
1477                batch_size,
1478                self.configuration.drop_last,
1479            )?;
1480        validate_distributed_rank(&plan_identity, self.configuration.rank)?;
1481        let expected_configuration =
1482            checkpoint_configuration(&plan_identity, &self.configuration, pin_memory_status)?;
1483        validate_plan_loader_state_envelope(&state, &expected_configuration)?;
1484
1485        self.dataset.validate_dataset_state(&state.dataset)?;
1486        self.plan.validate_checkpoint_state_with_batch_options(
1487            &state.sampler,
1488            state.epoch,
1489            state.next_batch,
1490            state.next_logical_sample,
1491            batch_size,
1492            self.configuration.drop_last,
1493        )?;
1494        let mut transform = self
1495            .transform_factory
1496            .create(None)
1497            .map_err(CheckpointBuildError::TransformFactory)?;
1498        transform.validate_snapshot(&state.transform)?;
1499        self.plan
1500            .validate_coordinator(&self.collator, &state.collate)?;
1501
1502        self.dataset.restore_dataset_validated(&state.dataset);
1503        self.plan
1504            .apply_batch_options(batch_size, self.configuration.drop_last);
1505        let batches = self.plan.restore_checkpoint_iter_validated(&state.sampler);
1506        transform.restore_validated(&state.transform);
1507        self.plan
1508            .restore_coordinator_validated(&mut self.collator, &state.collate);
1509
1510        Ok(OwnedDataLoader {
1511            dataset: Arc::new(self.dataset),
1512            plan: self.plan,
1513            collator: self.collator,
1514            transform_factory: Arc::new(self.transform_factory),
1515            worker_init: Arc::new(self.worker_init),
1516            workers: 0,
1517            persistent_workers: false,
1518            timeout: None,
1519            ordered: true,
1520            pin_memory_status,
1521            footprint: None,
1522            effective_prefetch_bytes: None,
1523            effective_prefetch: None,
1524            outstanding_capacity: None,
1525            loader_seed: self.configuration.loader_seed,
1526            rank: self.configuration.rank,
1527            next_generation: 0,
1528            persistent_pool: None,
1529            checkpoint: CheckpointActive {
1530                identity: requested_identity,
1531                configuration: expected_configuration,
1532                pending: Some(crate::checkpoint::PendingSerial {
1533                    batches,
1534                    transform,
1535                    next_batch: state.next_batch,
1536                    next_logical_sample: state.next_logical_sample,
1537                }),
1538            },
1539            policies: PhantomData,
1540        })
1541    }
1542}
1543
1544impl<D, S, C, F, I, X, M, N, K> DataLoaderBuilder<D, AutoBatch<S>, C, F, I, X, M, N, K>
1545where
1546    D: Dataset,
1547{
1548    /// Replaces the current sampler.
1549    pub fn sampler<S2>(
1550        mut self,
1551        sampler: S2,
1552    ) -> DataLoaderBuilder<D, AutoBatch<S2>, C, F, I, X, M, N, K> {
1553        self.explicit.sampler = true;
1554        self.map(|plan, collator| {
1555            (
1556                AutoBatch {
1557                    sampler,
1558                    batch_size: plan.batch_size,
1559                    drop_last: plan.drop_last,
1560                },
1561                collator,
1562            )
1563        })
1564    }
1565
1566    /// Replaces the current sampler with a seeded random sampler.
1567    ///
1568    /// # Errors
1569    ///
1570    /// Returns an invalid-configuration error for an empty dataset or an
1571    /// unallocatable permutation.
1572    #[allow(clippy::type_complexity)]
1573    pub fn shuffle(
1574        mut self,
1575        seed: u64,
1576    ) -> Result<DataLoaderBuilder<D, AutoBatch<RandomSampler>, C, F, I, X, M, N, K>> {
1577        self.explicit.shuffle = true;
1578        let sampler = RandomSampler::new(self.dataset.len(), seed)?;
1579        Ok(self.map(|plan, collator| {
1580            (
1581                AutoBatch {
1582                    sampler,
1583                    batch_size: plan.batch_size,
1584                    drop_last: plan.drop_last,
1585                },
1586                collator,
1587            )
1588        }))
1589    }
1590
1591    /// Disables automatic batching and selects default conversion.
1592    pub fn without_batching(
1593        mut self,
1594    ) -> DataLoaderBuilder<D, NoBatch<S, DefaultConverter>, C, F, I, X, M, N, K> {
1595        self.explicit.without_batching = true;
1596        self.map(|plan, collator| {
1597            (
1598                NoBatch {
1599                    sampler: plan.sampler,
1600                    converter: DefaultConverter,
1601                },
1602                collator,
1603            )
1604        })
1605    }
1606
1607    /// Replaces the automatic-batch collator.
1608    pub fn collate<C2>(
1609        self,
1610        collator: C2,
1611    ) -> DataLoaderBuilder<D, AutoBatch<S>, C2, F, I, X, M, N, K> {
1612        self.map(|plan, _| (plan, collator))
1613    }
1614}
1615
1616impl<D, B, C, F, I, X, M, N, K> DataLoaderBuilder<D, ExplicitBatches<B>, C, F, I, X, M, N, K>
1617where
1618    D: Dataset,
1619{
1620    /// Selects a sampler; build rejects its conflict with the earlier batch sampler.
1621    pub fn sampler<S>(
1622        mut self,
1623        sampler: S,
1624    ) -> DataLoaderBuilder<D, AutoBatch<S>, C, F, I, X, M, N, K> {
1625        self.explicit.sampler = true;
1626        let batch_size =
1627            NonZeroUsize::new(self.configuration.batch_size).unwrap_or(NonZeroUsize::MIN);
1628        let drop_last = self.configuration.drop_last;
1629        self.map(|_, collator| {
1630            (
1631                AutoBatch {
1632                    sampler,
1633                    batch_size,
1634                    drop_last,
1635                },
1636                collator,
1637            )
1638        })
1639    }
1640
1641    /// Selects shuffle; build rejects its conflict with the earlier batch sampler.
1642    ///
1643    /// # Errors
1644    ///
1645    /// Returns an invalid-configuration error for an empty dataset or an
1646    /// unallocatable permutation.
1647    #[allow(clippy::type_complexity)]
1648    pub fn shuffle(
1649        mut self,
1650        seed: u64,
1651    ) -> Result<DataLoaderBuilder<D, AutoBatch<RandomSampler>, C, F, I, X, M, N, K>> {
1652        self.explicit.shuffle = true;
1653        let sampler = RandomSampler::new(self.dataset.len(), seed)?;
1654        let batch_size =
1655            NonZeroUsize::new(self.configuration.batch_size).unwrap_or(NonZeroUsize::MIN);
1656        let drop_last = self.configuration.drop_last;
1657        Ok(self.map(|_, collator| {
1658            (
1659                AutoBatch {
1660                    sampler,
1661                    batch_size,
1662                    drop_last,
1663                },
1664                collator,
1665            )
1666        }))
1667    }
1668
1669    /// Disables batching; build rejects its conflict with the earlier batch sampler.
1670    pub fn without_batching(
1671        mut self,
1672    ) -> DataLoaderBuilder<D, NoBatch<SequentialSampler, DefaultConverter>, C, F, I, X, M, N, K>
1673    {
1674        self.explicit.without_batching = true;
1675        let length = self.dataset.len();
1676        self.map(|_, collator| {
1677            (
1678                NoBatch {
1679                    sampler: SequentialSampler::new(length),
1680                    converter: DefaultConverter,
1681                },
1682                collator,
1683            )
1684        })
1685    }
1686
1687    /// Replaces the explicit-batch collator.
1688    pub fn collate<C2>(
1689        self,
1690        collator: C2,
1691    ) -> DataLoaderBuilder<D, ExplicitBatches<B>, C2, F, I, X, M, N, K> {
1692        self.map(|plan, _| (plan, collator))
1693    }
1694}
1695
1696impl<D, S, V, C, F, I, X, M, N, K> DataLoaderBuilder<D, NoBatch<S, V>, C, F, I, X, M, N, K>
1697where
1698    D: Dataset,
1699{
1700    /// Replaces the no-batching sampler.
1701    pub fn sampler<S2>(
1702        mut self,
1703        sampler: S2,
1704    ) -> DataLoaderBuilder<D, NoBatch<S2, V>, C, F, I, X, M, N, K> {
1705        self.explicit.sampler = true;
1706        self.map(|plan, collator| {
1707            (
1708                NoBatch {
1709                    sampler,
1710                    converter: plan.converter,
1711                },
1712                collator,
1713            )
1714        })
1715    }
1716
1717    /// Replaces the no-batching sampler with a seeded random sampler.
1718    ///
1719    /// # Errors
1720    ///
1721    /// Returns an invalid-configuration error for an empty dataset or an
1722    /// unallocatable permutation.
1723    #[allow(clippy::type_complexity)]
1724    pub fn shuffle(
1725        mut self,
1726        seed: u64,
1727    ) -> Result<DataLoaderBuilder<D, NoBatch<RandomSampler, V>, C, F, I, X, M, N, K>> {
1728        self.explicit.shuffle = true;
1729        let sampler = RandomSampler::new(self.dataset.len(), seed)?;
1730        Ok(self.map(|plan, collator| {
1731            (
1732                NoBatch {
1733                    sampler,
1734                    converter: plan.converter,
1735                },
1736                collator,
1737            )
1738        }))
1739    }
1740
1741    /// Replaces the no-batching converter.
1742    pub fn convert<V2>(
1743        self,
1744        converter: V2,
1745    ) -> DataLoaderBuilder<D, NoBatch<S, V2>, C, F, I, X, M, N, K> {
1746        self.map(|plan, collator| {
1747            (
1748                NoBatch {
1749                    sampler: plan.sampler,
1750                    converter,
1751                },
1752                collator,
1753            )
1754        })
1755    }
1756}
1757
1758/// An owned, re-iterable map-style data loader.
1759///
1760/// The default [`SerialExecution`] capability accepts local non-`Send` data.
1761/// Calling [`DataLoaderBuilder::workers`] selects [`WorkerExecution`], where
1762/// positive counts share the dataset through [`Arc`]. Each worker owns a
1763/// bounded task lane and one transform instance; the coordinator alone owns
1764/// ordering and collation. This differs from PyTorch's process-local dataset
1765/// copies and worker-side collation while preserving bounded prefetch,
1766/// deterministic routing, task-local randomness, and ordered delivery.
1767#[allow(private_bounds)]
1768pub struct OwnedDataLoader<
1769    D,
1770    P,
1771    C,
1772    F = IdentityTransformFactory,
1773    I = NoWorkerInit,
1774    X = SerialExecution,
1775    M = MemoryDisabled,
1776    N = PinDisabled,
1777    K = CheckpointDisabled,
1778> where
1779    D: Dataset,
1780    F: TransformFactory<D::Sample>,
1781    I: WorkerInit,
1782    M: MemoryPolicy,
1783{
1784    dataset: Arc<D>,
1785    plan: P,
1786    collator: C,
1787    transform_factory: Arc<F>,
1788    worker_init: Arc<I>,
1789    workers: usize,
1790    persistent_workers: bool,
1791    timeout: Option<Duration>,
1792    ordered: bool,
1793    pin_memory_status: PinMemoryStatus,
1794    footprint: Option<FootprintFn<D, F>>,
1795    effective_prefetch_bytes: Option<NonZeroUsize>,
1796    effective_prefetch: Option<NonZeroUsize>,
1797    outstanding_capacity: Option<usize>,
1798    loader_seed: u64,
1799    rank: usize,
1800    next_generation: u64,
1801    persistent_pool: Option<WorkerPool<D, F, I, M>>,
1802    checkpoint: K,
1803    policies: PhantomData<(X, M, N)>,
1804}
1805
1806#[allow(private_bounds)]
1807impl<D, P, C, F, I, M, N> OwnedDataLoader<D, P, C, F, I, SerialExecution, M, N, CheckpointDisabled>
1808where
1809    D: Dataset,
1810    F: TransformFactory<D::Sample>,
1811    P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
1812    I: WorkerInit,
1813    M: MemoryPolicy,
1814    N: MapPinPolicy<D, P, C, F>,
1815{
1816    /// Starts a fresh finite iteration for the configured epoch.
1817    pub fn iter(&mut self) -> LoaderIter<'_, D, P, C, F, I, M, N, CheckpointDisabled> {
1818        let (transform, transform_error) = match self.transform_factory.create(None) {
1819            Ok(transform) => (Some(transform), None),
1820            Err(error) => (None, Some(error)),
1821        };
1822        let batches = transform.as_ref().map(|_| self.plan.iter());
1823        let epoch = self.plan.epoch();
1824        let run_context = WorkerRunContext::new(0, self.loader_seed, epoch);
1825        LoaderIter {
1826            dataset: self.dataset.as_ref(),
1827            plan: &mut self.plan,
1828            collator: &mut self.collator,
1829            batches,
1830            transform,
1831            transform_error,
1832            next_batch: 0,
1833            next_logical_sample: 0,
1834            exhausted: false,
1835            loader_seed: self.loader_seed,
1836            epoch,
1837            rank: self.rank,
1838            run_context,
1839            pin_memory_status: self.pin_memory_status,
1840            checkpoint: CheckpointDisabled,
1841            output: PhantomData,
1842            policies: PhantomData,
1843        }
1844    }
1845}
1846
1847#[allow(private_bounds, private_interfaces)]
1848impl<D, P, C, F, I, N>
1849    OwnedDataLoader<
1850        D,
1851        P,
1852        C,
1853        F,
1854        I,
1855        SerialExecution,
1856        MemoryDisabled,
1857        N,
1858        CheckpointActive<P::Iter, F::Transform>,
1859    >
1860where
1861    D: DatasetCheckpoint,
1862    F: TransformFactory<D::Sample>,
1863    F::Transform: WorkerCheckpoint,
1864    P: LoaderPlan<TransformOutput<D, F>, C> + CheckpointPlan<TransformOutput<D, F>, C>,
1865    I: WorkerInit,
1866    N: MapPinPolicy<D, P, C, F>,
1867{
1868    /// Starts the retained resumed cursor once, then fresh exact iterations.
1869    pub fn iter(
1870        &mut self,
1871    ) -> LoaderIter<'_, D, P, C, F, I, MemoryDisabled, N, CheckpointIteration> {
1872        let pending = self.checkpoint.pending.take();
1873        let (batches, transform, transform_error, next_batch, next_logical_sample) = match pending {
1874            Some(pending) => (
1875                Some(pending.batches),
1876                Some(pending.transform),
1877                None,
1878                pending.next_batch,
1879                pending.next_logical_sample,
1880            ),
1881            None => match self.transform_factory.create(None) {
1882                Ok(transform) => (Some(self.plan.iter()), Some(transform), None, 0, 0),
1883                Err(error) => (None, None, Some(error), 0, 0),
1884            },
1885        };
1886        let epoch = self.plan.epoch();
1887        let run_context = WorkerRunContext::new(0, self.loader_seed, epoch);
1888        LoaderIter {
1889            dataset: self.dataset.as_ref(),
1890            plan: &mut self.plan,
1891            collator: &mut self.collator,
1892            batches,
1893            transform,
1894            transform_error,
1895            next_batch,
1896            next_logical_sample,
1897            exhausted: false,
1898            loader_seed: self.loader_seed,
1899            epoch,
1900            rank: self.rank,
1901            run_context,
1902            pin_memory_status: self.pin_memory_status,
1903            checkpoint: CheckpointIteration {
1904                identity: self.checkpoint.identity.clone(),
1905                configuration: self.checkpoint.configuration.clone(),
1906                boundary_valid: true,
1907            },
1908            output: PhantomData,
1909            policies: PhantomData,
1910        }
1911    }
1912
1913    /// Selects the epoch used by future iterations.
1914    ///
1915    /// Calling this before the retained resumed cursor is consumed deliberately
1916    /// discards that cursor and its restored transform, so the selected epoch
1917    /// starts as an ordinary fresh iteration instead of mixing two epochs.
1918    pub fn set_epoch(&mut self, epoch: u64) {
1919        self.plan.set_epoch(epoch);
1920        self.checkpoint.pending = None;
1921    }
1922}
1923
1924#[allow(private_bounds)]
1925impl<D, P, C, F, I, X, M, N> OwnedDataLoader<D, P, C, F, I, X, M, N, CheckpointDisabled>
1926where
1927    D: Dataset,
1928    F: TransformFactory<D::Sample>,
1929    P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
1930    I: WorkerInit,
1931    M: MemoryPolicy,
1932{
1933    /// Selects the epoch used by future iterations.
1934    pub fn set_epoch(&mut self, epoch: u64) {
1935        self.plan.set_epoch(epoch);
1936    }
1937}
1938
1939#[allow(private_bounds)]
1940impl<D, P, C, F, I, X, M, N, K> OwnedDataLoader<D, P, C, F, I, X, M, N, K>
1941where
1942    D: Dataset,
1943    F: TransformFactory<D::Sample>,
1944    P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
1945    I: WorkerInit,
1946    M: MemoryPolicy,
1947{
1948    /// Returns the exact number of yielded items when the source is sized.
1949    pub fn len(&self) -> Option<usize> {
1950        self.plan.exact_len()
1951    }
1952
1953    /// Returns whether iteration is empty when the source is sized.
1954    pub fn is_empty(&self) -> Option<bool> {
1955        self.len().map(|length| length == 0)
1956    }
1957
1958    /// Returns the current epoch.
1959    pub fn epoch(&self) -> u64 {
1960        self.plan.epoch()
1961    }
1962
1963    /// Returns the configured worker count.
1964    pub fn workers(&self) -> usize {
1965        self.workers
1966    }
1967
1968    /// Returns the effective batches-prefetched-per-worker value.
1969    pub fn effective_prefetch_factor(&self) -> Option<NonZeroUsize> {
1970        self.effective_prefetch
1971    }
1972
1973    /// Returns whether results are configured for sampler order.
1974    pub fn is_ordered(&self) -> bool {
1975        self.ordered
1976    }
1977
1978    /// Returns whether pinning was requested.
1979    pub fn pin_memory_enabled(&self) -> bool {
1980        !matches!(self.pin_memory_status, PinMemoryStatus::Disabled)
1981    }
1982
1983    /// Returns the effective recursive pinning behavior.
1984    pub fn pin_memory_status(&self) -> PinMemoryStatus {
1985        self.pin_memory_status
1986    }
1987
1988    /// Returns the effective post-transform prefetch byte budget.
1989    pub fn effective_prefetch_bytes(&self) -> Option<NonZeroUsize> {
1990        self.effective_prefetch_bytes
1991    }
1992
1993    /// Returns the active timeout, if any.
1994    pub fn timeout(&self) -> Option<Duration> {
1995        self.timeout
1996    }
1997
1998    /// Returns whether persistent workers were requested.
1999    pub fn persistent_workers_enabled(&self) -> bool {
2000        self.persistent_workers
2001    }
2002}
2003
2004/// One fresh iteration borrowed from an [`OwnedDataLoader`].
2005pub struct LoaderIter<
2006    'a,
2007    D,
2008    P,
2009    C,
2010    F = IdentityTransformFactory,
2011    I = NoWorkerInit,
2012    M = MemoryDisabled,
2013    N = PinDisabled,
2014    K = CheckpointDisabled,
2015> where
2016    D: Dataset,
2017    F: TransformFactory<D::Sample>,
2018    P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2019    I: WorkerInit,
2020{
2021    dataset: &'a D,
2022    plan: &'a mut P,
2023    collator: &'a mut C,
2024    batches: Option<P::Iter>,
2025    transform: Option<F::Transform>,
2026    transform_error: Option<F::Error>,
2027    next_batch: u64,
2028    next_logical_sample: u64,
2029    exhausted: bool,
2030    loader_seed: u64,
2031    epoch: u64,
2032    rank: usize,
2033    run_context: WorkerRunContext,
2034    pin_memory_status: PinMemoryStatus,
2035    checkpoint: K,
2036    output: PhantomData<fn() -> I>,
2037    policies: PhantomData<(M, N)>,
2038}
2039
2040#[allow(private_bounds, private_interfaces)]
2041impl<D, P, C, F, I, N> LoaderIter<'_, D, P, C, F, I, MemoryDisabled, N, CheckpointIteration>
2042where
2043    D: DatasetCheckpoint,
2044    F: TransformFactory<D::Sample>,
2045    F::Transform: WorkerCheckpoint,
2046    P: LoaderPlan<TransformOutput<D, F>, C> + CheckpointPlan<TransformOutput<D, F>, C>,
2047    I: WorkerInit,
2048    N: MapPinPolicy<D, P, C, F>,
2049{
2050    /// Captures the exact next-visible-batch boundary.
2051    ///
2052    /// # Errors
2053    ///
2054    /// Returns [`LoaderError::Checkpoint`] after an error, end-of-input, or
2055    /// while a `next` call has advanced work that did not become visible.
2056    #[allow(clippy::type_complexity)]
2057    pub fn checkpoint(
2058        &mut self,
2059    ) -> std::result::Result<
2060        LoaderState<
2061            D::State,
2062            P::State,
2063            <F::Transform as WorkerCheckpoint>::State,
2064            P::CoordinatorState,
2065        >,
2066        IterError<D, P, C, F, I>,
2067    > {
2068        if !self.checkpoint.boundary_valid {
2069            return Err(LoaderError::Checkpoint {
2070                reason: "the active iterator is not at a consumer-visible batch boundary"
2071                    .to_owned(),
2072            });
2073        }
2074        let transform = self
2075            .transform
2076            .as_ref()
2077            .ok_or_else(|| LoaderError::Checkpoint {
2078                reason: "the serial transform was not constructed".to_owned(),
2079            })?;
2080        let sampler = self
2081            .plan
2082            .checkpoint_state(self.next_batch, self.next_logical_sample)
2083            .map_err(LoaderError::Configuration)?;
2084        Ok(LoaderState {
2085            schema_version: LOADER_STATE_SCHEMA_VERSION,
2086            dataset_identity: self.checkpoint.identity.clone(),
2087            epoch: self.epoch,
2088            iterator_generation: 0,
2089            next_batch: self.next_batch,
2090            next_logical_sample: self.next_logical_sample,
2091            dataset: self.dataset.snapshot_dataset(),
2092            sampler,
2093            transform: transform.snapshot(),
2094            collate: self.plan.save_coordinator(self.collator),
2095            rng_derivation_version: TASK_RNG_DERIVATION_VERSION,
2096            worker_seed_derivation_version: WORKER_SEED_DERIVATION_VERSION,
2097            configuration: self.checkpoint.configuration.clone(),
2098        })
2099    }
2100}
2101
2102impl<D, P, C, F, I, M, N, K> Iterator for LoaderIter<'_, D, P, C, F, I, M, N, K>
2103where
2104    D: Dataset,
2105    F: TransformFactory<D::Sample>,
2106    P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2107    I: WorkerInit,
2108    N: MapPinPolicy<D, P, C, F>,
2109    K: SerialBoundaryPolicy,
2110{
2111    type Item = std::result::Result<
2112        P::Batch,
2113        LoaderError<
2114            PipelineError<
2115                D::Error,
2116                <F::Transform as Transform<D::Sample>>::Error,
2117                P::Error,
2118                F::Error,
2119                I::Error,
2120            >,
2121        >,
2122    >;
2123
2124    fn next(&mut self) -> Option<Self::Item> {
2125        if self.exhausted {
2126            return None;
2127        }
2128        self.checkpoint.begin_next();
2129        if let Some(source) = self.transform_error.take() {
2130            self.exhausted = true;
2131            return Some(Err(LoaderError::Pipeline {
2132                batch: None,
2133                worker: None,
2134                source: PipelineError::TransformInit(source),
2135            }));
2136        }
2137
2138        let indices = match self.batches.as_mut().and_then(Iterator::next) {
2139            Some(indices) => indices,
2140            None => {
2141                self.exhausted = true;
2142                return None;
2143            }
2144        };
2145        let batch = self.next_batch;
2146        let samples = match self.dataset.get_batch(&indices) {
2147            Ok(samples) => samples,
2148            Err(source) => {
2149                self.exhausted = true;
2150                return Some(Err(LoaderError::Pipeline {
2151                    batch: Some(batch),
2152                    worker: None,
2153                    source: PipelineError::Dataset(source),
2154                }));
2155            }
2156        };
2157        if samples.len() != indices.len() {
2158            self.exhausted = true;
2159            return Some(Err(LoaderError::InvalidBatchCardinality {
2160                batch,
2161                worker: None,
2162                expected: indices.len(),
2163                actual: samples.len(),
2164            }));
2165        }
2166        let mut transformed = Vec::with_capacity(samples.len());
2167        for sample in samples {
2168            let context = TaskContext {
2169                loader_seed: self.loader_seed,
2170                epoch: self.epoch,
2171                rank: self.rank,
2172                logical_sample: self.next_logical_sample,
2173                stage: 0,
2174                cancellation: self.run_context.cancellation.clone(),
2175                deadline: self.run_context.deadline.clone(),
2176            };
2177            let Some(transform) = self.transform.as_mut() else {
2178                self.exhausted = true;
2179                return None;
2180            };
2181            match transform.transform(sample, &context) {
2182                Ok(sample) => transformed.push(sample),
2183                Err(source) => {
2184                    self.exhausted = true;
2185                    return Some(Err(LoaderError::Pipeline {
2186                        batch: Some(batch),
2187                        worker: None,
2188                        source: PipelineError::Transform(source),
2189                    }));
2190                }
2191            }
2192            match self.next_logical_sample.checked_add(1) {
2193                Some(next) => self.next_logical_sample = next,
2194                None => self.exhausted = true,
2195            }
2196        }
2197        let result =
2198            self.plan
2199                .finish(self.collator, transformed)
2200                .map_err(|source| LoaderError::Pipeline {
2201                    batch: Some(batch),
2202                    worker: None,
2203                    source: PipelineError::Collate(source),
2204                })
2205                .and_then(|value| match self.pin_memory_status {
2206                    PinMemoryStatus::Enabled(device) => N::pin(value, device)
2207                        .map_err(|source| LoaderError::PinMemory { batch, source }),
2208                    PinMemoryStatus::Disabled | PinMemoryStatus::DisabledNoAccelerator => Ok(value),
2209                });
2210        if result.is_err() {
2211            self.exhausted = true;
2212        } else if let Some(next) = self.next_batch.checked_add(1) {
2213            self.next_batch = next;
2214            self.checkpoint.commit_visible();
2215        } else {
2216            self.exhausted = true;
2217        }
2218        Some(result)
2219    }
2220}
2221
2222/// One positive-worker iteration borrowed from an [`OwnedDataLoader`].
2223#[allow(private_bounds)]
2224pub struct WorkerLoaderIter<
2225    'a,
2226    D,
2227    P,
2228    C,
2229    F = IdentityTransformFactory,
2230    I = NoWorkerInit,
2231    M = MemoryDisabled,
2232    N = PinDisabled,
2233    K = CheckpointDisabled,
2234> where
2235    D: Dataset,
2236    F: TransformFactory<D::Sample>,
2237    P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2238    I: WorkerInit,
2239    M: MemoryPolicy,
2240{
2241    dataset: Arc<D>,
2242    plan: &'a mut P,
2243    collator: &'a mut C,
2244    batches: Option<P::Iter>,
2245    serial_transform: Option<F::Transform>,
2246    serial_transform_error: Option<F::Error>,
2247    pool: Option<IteratorPool<'a, D, F, I, M>>,
2248    completed: BTreeMap<u64, CompletedBatch<D, F, M>>,
2249    pending_error: Option<IterError<D, P, C, F, I>>,
2250    generation: u64,
2251    run_context: Option<WorkerRunContext>,
2252    workers: usize,
2253    capacity: usize,
2254    ordered: bool,
2255    timeout: Option<Duration>,
2256    loader_seed: u64,
2257    epoch: u64,
2258    rank: usize,
2259    next_submission: u64,
2260    next_visible: u64,
2261    next_logical_sample: u64,
2262    pending_submission: Option<PendingSubmission>,
2263    outstanding: usize,
2264    source_exhausted: bool,
2265    submission_closed: bool,
2266    exhausted: bool,
2267    pin_memory_status: PinMemoryStatus,
2268    _checkpoint: K,
2269    policies: PhantomData<(M, N)>,
2270}
2271
2272enum IteratorPool<'a, D, F, I, M>
2273where
2274    D: Dataset,
2275    F: TransformFactory<D::Sample>,
2276    I: WorkerInit,
2277    M: MemoryPolicy,
2278{
2279    Owned(WorkerPool<D, F, I, M>),
2280    Persistent {
2281        pool: &'a mut WorkerPool<D, F, I, M>,
2282        generation: u64,
2283    },
2284}
2285
2286impl<D, F, I, M> IteratorPool<'_, D, F, I, M>
2287where
2288    D: Dataset,
2289    F: TransformFactory<D::Sample>,
2290    I: WorkerInit,
2291    M: MemoryPolicy,
2292{
2293    fn pool(&self) -> &WorkerPool<D, F, I, M> {
2294        match self {
2295            Self::Owned(pool) => pool,
2296            Self::Persistent { pool, .. } => pool,
2297        }
2298    }
2299
2300    fn pool_mut(&mut self) -> &mut WorkerPool<D, F, I, M> {
2301        match self {
2302            Self::Owned(pool) => pool,
2303            Self::Persistent { pool, .. } => pool,
2304        }
2305    }
2306
2307    fn is_persistent(&self) -> bool {
2308        matches!(self, Self::Persistent { .. })
2309    }
2310}
2311
2312impl<D, F, I, M> Drop for IteratorPool<'_, D, F, I, M>
2313where
2314    D: Dataset,
2315    F: TransformFactory<D::Sample>,
2316    I: WorkerInit,
2317    M: MemoryPolicy,
2318{
2319    fn drop(&mut self) {
2320        if let Self::Persistent { pool, generation } = self
2321            && pool.quiesce(*generation).is_err()
2322        {
2323            pool.poison();
2324        }
2325    }
2326}
2327
2328#[allow(private_bounds)]
2329impl<D, P, C, F, I, M, N> OwnedDataLoader<D, P, C, F, I, WorkerExecution, M, N, CheckpointDisabled>
2330where
2331    D: Dataset + Send + Sync + 'static,
2332    D::Sample: Send + 'static,
2333    D::Error: Send + 'static,
2334    F: TransformFactory<D::Sample> + Send + Sync + 'static,
2335    F::Transform: Send + 'static,
2336    <F::Transform as Transform<D::Sample>>::Output: Send + 'static,
2337    <F::Transform as Transform<D::Sample>>::Error: Send + 'static,
2338    F::Error: Send + 'static,
2339    P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2340    I: WorkerInit + Send + Sync + 'static,
2341    I::Error: Send + 'static,
2342    M: MemoryPolicy + Send + 'static,
2343    N: MapPinPolicy<D, P, C, F>,
2344{
2345    /// Starts a fresh generation on a bounded worker pool, or the explicit
2346    /// zero-worker path.
2347    pub fn iter(&mut self) -> WorkerLoaderIter<'_, D, P, C, F, I, M, N, CheckpointDisabled> {
2348        let (epoch, epoch_panicked) = if self.workers == 0 {
2349            (self.plan.epoch(), false)
2350        } else {
2351            match catch_unwind(AssertUnwindSafe(|| self.plan.epoch())) {
2352                Ok(epoch) => (epoch, false),
2353                Err(_) => (0, true),
2354            }
2355        };
2356        let mut iterator = WorkerLoaderIter {
2357            dataset: Arc::clone(&self.dataset),
2358            plan: &mut self.plan,
2359            collator: &mut self.collator,
2360            batches: None,
2361            serial_transform: None,
2362            serial_transform_error: None,
2363            pool: None,
2364            completed: BTreeMap::new(),
2365            pending_error: None,
2366            generation: self.next_generation,
2367            run_context: Some(WorkerRunContext::new(
2368                self.next_generation,
2369                self.loader_seed,
2370                epoch,
2371            )),
2372            workers: self.workers,
2373            capacity: self.outstanding_capacity.unwrap_or(0),
2374            ordered: self.ordered,
2375            timeout: self.timeout,
2376            loader_seed: self.loader_seed,
2377            epoch,
2378            rank: self.rank,
2379            next_submission: 0,
2380            next_visible: 0,
2381            next_logical_sample: 0,
2382            pending_submission: None,
2383            outstanding: 0,
2384            source_exhausted: false,
2385            submission_closed: false,
2386            exhausted: false,
2387            pin_memory_status: self.pin_memory_status,
2388            _checkpoint: CheckpointDisabled,
2389            policies: PhantomData,
2390        };
2391
2392        if epoch_panicked {
2393            iterator.pending_error = Some(LoaderError::CoordinatorPanic {
2394                stage: P::EPOCH_PANIC_STAGE,
2395                batch: None,
2396            });
2397            return iterator;
2398        }
2399        if self.workers == 0 {
2400            match self.transform_factory.create(None) {
2401                Ok(transform) => {
2402                    iterator.serial_transform = Some(transform);
2403                    iterator.batches = Some(iterator.plan.iter());
2404                }
2405                Err(error) => iterator.serial_transform_error = Some(error),
2406            }
2407            return iterator;
2408        }
2409        let Some(next_generation) = self.next_generation.checked_add(1) else {
2410            iterator.pending_error = Some(LoaderError::Configuration(invalid_configuration(
2411                "workers",
2412                "iterator generation overflowed",
2413            )));
2414            return iterator;
2415        };
2416        iterator.batches = match catch_unwind(AssertUnwindSafe(|| iterator.plan.iter())) {
2417            Ok(batches) => Some(batches),
2418            Err(_) => {
2419                iterator.pending_error = Some(LoaderError::CoordinatorPanic {
2420                    stage: P::CREATION_PANIC_STAGE,
2421                    batch: None,
2422                });
2423                return iterator;
2424            }
2425        };
2426        self.next_generation = next_generation;
2427        let configuration = WorkerPoolConfiguration {
2428            workers: self.workers,
2429            prefetch_factor: self
2430                .effective_prefetch
2431                .expect("positive workers have a validated prefetch factor")
2432                .get(),
2433            result_capacity: iterator.capacity,
2434            seed_generation: iterator.generation,
2435            loader_seed: self.loader_seed,
2436            rank: self.rank,
2437        };
2438        let run_context =
2439            WorkerRunContext::new(iterator.generation, self.loader_seed, iterator.epoch)
2440                .with_byte_budget(
2441                    self.effective_prefetch_bytes
2442                        .map(|limit| ByteBudget::new(limit.get())),
2443                );
2444        iterator.run_context = Some(run_context.clone());
2445
2446        if self.persistent_workers {
2447            if self.persistent_pool.is_none() {
2448                match WorkerPool::new(
2449                    Arc::clone(&self.dataset),
2450                    Arc::clone(&self.transform_factory),
2451                    Arc::clone(&self.worker_init),
2452                    configuration,
2453                    self.footprint,
2454                    self.ordered,
2455                ) {
2456                    Ok(pool) => self.persistent_pool = Some(pool),
2457                    Err(error) => {
2458                        iterator.pending_error = Some(LoaderError::Configuration(error));
2459                        return iterator;
2460                    }
2461                }
2462            }
2463            let pool = self
2464                .persistent_pool
2465                .as_mut()
2466                .expect("persistent pool was created");
2467            if pool.is_poisoned() || pool.start_generation(run_context).is_err() {
2468                iterator.pending_error = Some(LoaderError::ChannelClosed { batch: 0 });
2469                return iterator;
2470            }
2471            iterator.pool = Some(IteratorPool::Persistent {
2472                pool,
2473                generation: iterator.generation,
2474            });
2475        } else {
2476            match WorkerPool::new(
2477                Arc::clone(&self.dataset),
2478                Arc::clone(&self.transform_factory),
2479                Arc::clone(&self.worker_init),
2480                configuration,
2481                self.footprint,
2482                self.ordered,
2483            ) {
2484                Ok(mut pool) => {
2485                    if pool.start_generation(run_context).is_err() {
2486                        iterator.pending_error = Some(LoaderError::ChannelClosed { batch: 0 });
2487                    } else {
2488                        iterator.pool = Some(IteratorPool::Owned(pool));
2489                    }
2490                }
2491                Err(error) => iterator.pending_error = Some(LoaderError::Configuration(error)),
2492            }
2493        }
2494        if iterator.pending_error.is_none() {
2495            iterator.fill_available();
2496        }
2497        iterator
2498    }
2499}
2500
2501#[allow(private_bounds)]
2502impl<D, P, C, F, I, M, N, K> WorkerLoaderIter<'_, D, P, C, F, I, M, N, K>
2503where
2504    D: Dataset + Send + Sync + 'static,
2505    D::Sample: Send + 'static,
2506    D::Error: Send + 'static,
2507    F: TransformFactory<D::Sample> + Send + Sync + 'static,
2508    F::Transform: Send + 'static,
2509    <F::Transform as Transform<D::Sample>>::Output: Send + 'static,
2510    <F::Transform as Transform<D::Sample>>::Error: Send + 'static,
2511    F::Error: Send + 'static,
2512    P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2513    I: WorkerInit + Send + Sync + 'static,
2514    I::Error: Send + 'static,
2515    M: MemoryPolicy + Send + 'static,
2516    N: MapPinPolicy<D, P, C, F>,
2517    K: WorkerBoundaryPolicy<IterError<D, P, C, F, I>>,
2518{
2519    fn fill_available(&mut self) {
2520        while self.outstanding < self.capacity && !self.source_exhausted && !self.submission_closed
2521        {
2522            if !self.submit_one() {
2523                break;
2524            }
2525        }
2526    }
2527
2528    fn submit_one(&mut self) -> bool {
2529        if let Some(pending) = self.pending_submission.take() {
2530            return self.try_submit(pending);
2531        }
2532        let next_batch = catch_unwind(AssertUnwindSafe(|| {
2533            self.batches.as_mut().and_then(Iterator::next)
2534        }));
2535        let Some(indices) = (match next_batch {
2536            Ok(indices) => indices,
2537            Err(_) => {
2538                self.pending_error = Some(LoaderError::CoordinatorPanic {
2539                    stage: P::REFILL_PANIC_STAGE,
2540                    batch: Some(self.next_submission),
2541                });
2542                self.submission_closed = true;
2543                return false;
2544            }
2545        }) else {
2546            self.source_exhausted = true;
2547            return false;
2548        };
2549        let batch_sequence = self.next_submission;
2550        let Some(next_submission) = batch_sequence.checked_add(1) else {
2551            self.pending_error = Some(LoaderError::Configuration(invalid_configuration(
2552                "sampler",
2553                "batch sequence overflowed",
2554            )));
2555            self.submission_closed = true;
2556            return false;
2557        };
2558        let logical_start = self.next_logical_sample;
2559        let Some(logical_end) = logical_start.checked_add(indices.len() as u64) else {
2560            self.pending_error = Some(LoaderError::Configuration(invalid_configuration(
2561                "sampler",
2562                "logical sample identifier overflowed",
2563            )));
2564            self.submission_closed = true;
2565            return false;
2566        };
2567        let logical_samples = (logical_start..logical_end).collect();
2568        let worker = batch_sequence as usize % self.workers;
2569        self.try_submit(PendingSubmission {
2570            worker,
2571            task: WorkerTask {
2572                generation: self.generation,
2573                batch_sequence,
2574                logical_samples,
2575                indices,
2576            },
2577            next_submission,
2578            logical_end,
2579        })
2580    }
2581
2582    fn try_submit(&mut self, pending: PendingSubmission) -> bool {
2583        let PendingSubmission {
2584            worker,
2585            task,
2586            next_submission,
2587            logical_end,
2588        } = pending;
2589        match self
2590            .pool
2591            .as_ref()
2592            .expect("submissions require an active worker pool")
2593            .pool()
2594            .submit(worker, task)
2595        {
2596            WorkerSubmit::Submitted => {
2597                self._checkpoint.submitted(next_submission - 1, logical_end);
2598                self.next_submission = next_submission;
2599                self.next_logical_sample = logical_end;
2600                self.outstanding += 1;
2601                true
2602            }
2603            WorkerSubmit::Full(task) => {
2604                self.pending_submission = Some(PendingSubmission {
2605                    worker,
2606                    task,
2607                    next_submission,
2608                    logical_end,
2609                });
2610                false
2611            }
2612            WorkerSubmit::Closed => {
2613                self.submission_closed = true;
2614                false
2615            }
2616        }
2617    }
2618
2619    fn stop(&mut self) {
2620        self.stop_inner(false);
2621    }
2622
2623    fn stop_poisoned(&mut self) {
2624        self.stop_inner(true);
2625    }
2626
2627    fn stop_inner(&mut self, poisoned: bool) {
2628        self.exhausted = true;
2629        if let Some(run_context) = &self.run_context {
2630            run_context.cancel();
2631        }
2632        if let Some(mut pool) = self.pool.take() {
2633            let should_poison = poisoned
2634                || (pool.is_persistent() && pool.pool_mut().quiesce(self.generation).is_err());
2635            if should_poison {
2636                pool.pool_mut().poison();
2637            }
2638        }
2639        self.completed.clear();
2640        self.pending_submission = None;
2641    }
2642
2643    fn next_serial(&mut self) -> Option<IterResult<D, P, C, F, I>> {
2644        if let Some(source) = self.serial_transform_error.take() {
2645            self.exhausted = true;
2646            return Some(Err(LoaderError::Pipeline {
2647                batch: None,
2648                worker: None,
2649                source: PipelineError::TransformInit(source),
2650            }));
2651        }
2652        let indices = match self.batches.as_mut().and_then(Iterator::next) {
2653            Some(indices) => indices,
2654            None => {
2655                self.exhausted = true;
2656                return None;
2657            }
2658        };
2659        let batch = self.next_visible;
2660        let samples = match self.dataset.get_batch(&indices) {
2661            Ok(samples) => samples,
2662            Err(source) => {
2663                self.exhausted = true;
2664                return Some(Err(LoaderError::Pipeline {
2665                    batch: Some(batch),
2666                    worker: None,
2667                    source: PipelineError::Dataset(source),
2668                }));
2669            }
2670        };
2671        if samples.len() != indices.len() {
2672            self.exhausted = true;
2673            return Some(Err(LoaderError::InvalidBatchCardinality {
2674                batch,
2675                worker: None,
2676                expected: indices.len(),
2677                actual: samples.len(),
2678            }));
2679        }
2680        let mut transformed = Vec::with_capacity(samples.len());
2681        for sample in samples {
2682            let context = TaskContext {
2683                loader_seed: self.loader_seed,
2684                epoch: self.epoch,
2685                rank: self.rank,
2686                logical_sample: self.next_logical_sample,
2687                stage: 0,
2688                cancellation: self
2689                    .run_context
2690                    .as_ref()
2691                    .expect("serial generation context")
2692                    .cancellation
2693                    .clone(),
2694                deadline: self
2695                    .run_context
2696                    .as_ref()
2697                    .expect("serial generation context")
2698                    .deadline
2699                    .clone(),
2700            };
2701            let transform = self
2702                .serial_transform
2703                .as_mut()
2704                .expect("serial transform exists after successful construction");
2705            match transform.transform(sample, &context) {
2706                Ok(sample) => transformed.push(sample),
2707                Err(source) => {
2708                    self.exhausted = true;
2709                    return Some(Err(LoaderError::Pipeline {
2710                        batch: Some(batch),
2711                        worker: None,
2712                        source: PipelineError::Transform(source),
2713                    }));
2714                }
2715            }
2716            self.next_logical_sample += 1;
2717        }
2718        let result =
2719            self.plan
2720                .finish(self.collator, transformed)
2721                .map_err(|source| LoaderError::Pipeline {
2722                    batch: Some(batch),
2723                    worker: None,
2724                    source: PipelineError::Collate(source),
2725                })
2726                .and_then(|value| match self.pin_memory_status {
2727                    PinMemoryStatus::Enabled(device) => N::pin(value, device)
2728                        .map_err(|source| LoaderError::PinMemory { batch, source }),
2729                    PinMemoryStatus::Disabled | PinMemoryStatus::DisabledNoAccelerator => Ok(value),
2730                });
2731        if result.is_err() {
2732            self.exhausted = true;
2733        } else {
2734            self._checkpoint
2735                .committed(self.next_visible, self.next_logical_sample);
2736            self.next_visible += 1;
2737        }
2738        Some(result)
2739    }
2740
2741    fn publish(
2742        &mut self,
2743        worker: usize,
2744        batch: WorkerBatch<
2745            <F::Transform as Transform<D::Sample>>::Output,
2746            <M as MemoryPolicy>::Permit,
2747        >,
2748    ) -> IterResult<D, P, C, F, I> {
2749        debug_assert_eq!(batch.generation, self.generation);
2750        self.outstanding -= 1;
2751        let sequence = batch.batch_sequence;
2752        let WorkerBatch {
2753            batch_sequence,
2754            samples,
2755            permit,
2756            ..
2757        } = batch;
2758        drop(permit);
2759        let result = match catch_unwind(AssertUnwindSafe(|| {
2760            self.plan.finish(self.collator, samples)
2761        })) {
2762            Ok(result) => result.map_err(|source| LoaderError::Pipeline {
2763                batch: Some(sequence),
2764                worker: None,
2765                source: PipelineError::Collate(source),
2766            }),
2767            Err(_) => Err(LoaderError::CoordinatorPanic {
2768                stage: P::FINISH_PANIC_STAGE,
2769                batch: Some(sequence),
2770            }),
2771        };
2772        let result = result.and_then(|value| match self.pin_memory_status {
2773            PinMemoryStatus::Enabled(device) => {
2774                N::pin(value, device).map_err(|source| LoaderError::PinMemory {
2775                    batch: batch_sequence,
2776                    source,
2777                })
2778            }
2779            PinMemoryStatus::Disabled | PinMemoryStatus::DisabledNoAccelerator => Ok(value),
2780        });
2781        if result.is_ok() {
2782            self._checkpoint
2783                .committed(sequence, self.next_logical_sample);
2784            self.next_visible = self.next_visible.saturating_add(1);
2785            self.fill_available();
2786        } else {
2787            self.stop();
2788        }
2789        let _ = worker;
2790        result
2791    }
2792
2793    fn map_failure(
2794        &mut self,
2795        worker: usize,
2796        batch: Option<u64>,
2797        failure: WorkerStageFailure<D, F, I>,
2798    ) -> IterError<D, P, C, F, I> {
2799        match failure {
2800            WorkerFailure::Dataset(source) => LoaderError::Pipeline {
2801                batch,
2802                worker: Some(worker),
2803                source: PipelineError::Dataset(source),
2804            },
2805            WorkerFailure::Transform(source) => LoaderError::Pipeline {
2806                batch,
2807                worker: Some(worker),
2808                source: PipelineError::Transform(source),
2809            },
2810            WorkerFailure::TransformInit(source) => LoaderError::Pipeline {
2811                batch,
2812                worker: Some(worker),
2813                source: PipelineError::TransformInit(source),
2814            },
2815            WorkerFailure::WorkerInit(source) => LoaderError::Pipeline {
2816                batch,
2817                worker: Some(worker),
2818                source: PipelineError::WorkerInit(source),
2819            },
2820            WorkerFailure::InvalidBatchCardinality { expected, actual } => {
2821                LoaderError::InvalidBatchCardinality {
2822                    batch: batch.unwrap_or(self.next_visible),
2823                    worker: Some(worker),
2824                    expected,
2825                    actual,
2826                }
2827            }
2828            WorkerFailure::MemoryLimit { limit, actual } => LoaderError::MemoryLimit {
2829                batch,
2830                worker: Some(worker),
2831                sequence: None,
2832                logical_id: None,
2833                limit,
2834                actual,
2835            },
2836            WorkerFailure::Panic => LoaderError::WorkerPanic { worker, batch },
2837        }
2838    }
2839}
2840
2841impl<D, P, C, F, I, M, N, K> Iterator for WorkerLoaderIter<'_, D, P, C, F, I, M, N, K>
2842where
2843    D: Dataset + Send + Sync + 'static,
2844    D::Sample: Send + 'static,
2845    D::Error: Send + 'static,
2846    F: TransformFactory<D::Sample> + Send + Sync + 'static,
2847    F::Transform: Send + 'static,
2848    <F::Transform as Transform<D::Sample>>::Output: Send + 'static,
2849    <F::Transform as Transform<D::Sample>>::Error: Send + 'static,
2850    F::Error: Send + 'static,
2851    P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2852    I: WorkerInit + Send + Sync + 'static,
2853    I::Error: Send + 'static,
2854    M: MemoryPolicy + Send + 'static,
2855    N: MapPinPolicy<D, P, C, F>,
2856    K: WorkerBoundaryPolicy<IterError<D, P, C, F, I>>,
2857{
2858    type Item = std::result::Result<
2859        P::Batch,
2860        LoaderError<
2861            PipelineError<
2862                D::Error,
2863                <F::Transform as Transform<D::Sample>>::Error,
2864                P::Error,
2865                F::Error,
2866                I::Error,
2867            >,
2868        >,
2869    >;
2870
2871    fn next(&mut self) -> Option<Self::Item> {
2872        self._checkpoint.begin_next();
2873        if self.exhausted {
2874            return None;
2875        }
2876        if let Some(error) = self.pending_error.take() {
2877            self.stop();
2878            return Some(Err(error));
2879        }
2880        if self.workers == 0 {
2881            return self.next_serial();
2882        }
2883        let mut deadline_armed = false;
2884        loop {
2885            if let Some(error) = self._checkpoint.take_error(self.next_visible) {
2886                self.stop();
2887                return Some(Err(error));
2888            }
2889            if self.ordered
2890                && let Some((worker, batch)) = self.completed.remove(&self.next_visible)
2891            {
2892                if deadline_armed {
2893                    self.run_context
2894                        .as_ref()
2895                        .expect("worker context")
2896                        .deadline
2897                        .disarm();
2898                }
2899                return Some(self.publish(worker, batch));
2900            }
2901            if self.outstanding == 0 && self.source_exhausted {
2902                if deadline_armed {
2903                    self.run_context
2904                        .as_ref()
2905                        .expect("worker context")
2906                        .deadline
2907                        .disarm();
2908                }
2909                self.stop();
2910                return None;
2911            }
2912            let run_context = self
2913                .run_context
2914                .as_ref()
2915                .expect("worker iteration has a generation context")
2916                .clone();
2917            if !deadline_armed && let Some(timeout) = self.timeout {
2918                run_context.deadline.arm(timeout);
2919                deadline_armed = true;
2920            }
2921            let remaining = self.timeout.and_then(|_| run_context.deadline.remaining());
2922            let received = self
2923                .pool
2924                .as_ref()
2925                .expect("worker iteration has an active pool")
2926                .pool()
2927                .receive(&run_context, remaining);
2928            let completion = match received {
2929                WorkerReceive::Completion(completion) => completion,
2930                WorkerReceive::Timeout => {
2931                    let batch = self.next_visible;
2932                    run_context.cancel();
2933                    run_context.deadline.disarm();
2934                    self.stop();
2935                    return Some(Err(LoaderError::Timeout { batch }));
2936                }
2937                WorkerReceive::Cancelled => {
2938                    run_context.deadline.disarm();
2939                    self.stop();
2940                    return Some(Err(LoaderError::Cancelled));
2941                }
2942                WorkerReceive::Closed => {
2943                    let batch = self.next_visible;
2944                    run_context.deadline.disarm();
2945                    self.stop_poisoned();
2946                    return Some(Err(LoaderError::ChannelClosed { batch }));
2947                }
2948            };
2949            if completion.generation != self.generation {
2950                continue;
2951            }
2952            match completion.result {
2953                Err(failure) => {
2954                    let poisons_pool = matches!(
2955                        &failure,
2956                        WorkerFailure::TransformInit(_)
2957                            | WorkerFailure::WorkerInit(_)
2958                            | WorkerFailure::Panic
2959                    );
2960                    let error =
2961                        self.map_failure(completion.worker, completion.batch_sequence, failure);
2962                    if !poisons_pool && let Some(sequence) = completion.batch_sequence {
2963                        match self._checkpoint.defer_error(sequence, error) {
2964                            None => continue,
2965                            Some(error) => {
2966                                run_context.deadline.disarm();
2967                                self.stop();
2968                                return Some(Err(error));
2969                            }
2970                        }
2971                    }
2972                    run_context.deadline.disarm();
2973                    if poisons_pool {
2974                        self.stop_poisoned();
2975                    } else {
2976                        self.stop();
2977                    }
2978                    return Some(Err(error));
2979                }
2980                Ok(WorkerMessage::Batch(batch))
2981                    if self.ordered && batch.batch_sequence != self.next_visible =>
2982                {
2983                    self.completed
2984                        .insert(batch.batch_sequence, (completion.worker, batch));
2985                }
2986                Ok(WorkerMessage::Batch(batch)) => {
2987                    run_context.deadline.disarm();
2988                    return Some(self.publish(completion.worker, batch));
2989                }
2990                Ok(WorkerMessage::Quiesced) => continue,
2991            }
2992        }
2993    }
2994}
2995
2996fn validate_configuration(
2997    configuration: &BuilderConfiguration,
2998    explicit: ExplicitArguments,
2999) -> Result<()> {
3000    if configuration.batch_size == 0 {
3001        return Err(invalid_configuration(
3002            "batch_size",
3003            "must be greater than zero",
3004        ));
3005    }
3006    if explicit.sampler && explicit.shuffle {
3007        return Err(invalid_configuration(
3008            "sampler",
3009            "cannot be combined with shuffle",
3010        ));
3011    }
3012    if explicit.batch_sampler
3013        && (explicit.batch_size
3014            || explicit.shuffle
3015            || explicit.sampler
3016            || explicit.drop_last
3017            || explicit.without_batching)
3018    {
3019        return Err(invalid_configuration(
3020            "batch_sampler",
3021            "cannot be combined with batch_size, shuffle, sampler, drop_last, or no batching",
3022        ));
3023    }
3024    if explicit.without_batching && explicit.drop_last && configuration.drop_last {
3025        return Err(invalid_configuration(
3026            "drop_last",
3027            "cannot be enabled when automatic batching is disabled",
3028        ));
3029    }
3030    if configuration.workers == 0 && configuration.timeout.is_some() {
3031        return Err(invalid_configuration(
3032            "timeout",
3033            "requires a positive worker count",
3034        ));
3035    }
3036    if configuration
3037        .timeout
3038        .is_some_and(|timeout| !Deadline::can_represent(timeout))
3039    {
3040        return Err(invalid_configuration(
3041            "timeout",
3042            "exceeds the platform monotonic clock range",
3043        ));
3044    }
3045    if explicit.prefetch_factor && configuration.workers == 0 {
3046        return Err(invalid_configuration(
3047            "prefetch_factor",
3048            "requires a positive worker count",
3049        ));
3050    }
3051    if explicit.prefetch_bytes && configuration.workers == 0 {
3052        return Err(invalid_configuration(
3053            "prefetch_bytes",
3054            "requires a positive worker count",
3055        ));
3056    }
3057    if configuration.prefetch_factor == Some(0) {
3058        return Err(invalid_configuration(
3059            "prefetch_factor",
3060            "must be greater than zero",
3061        ));
3062    }
3063    if configuration.persistent_workers && configuration.workers == 0 {
3064        return Err(invalid_configuration(
3065            "persistent_workers",
3066            "requires a positive worker count",
3067        ));
3068    }
3069    Ok(())
3070}
3071
3072fn validate_exact_builder(
3073    configuration: &BuilderConfiguration,
3074    explicit: ExplicitArguments,
3075    identity: &str,
3076) -> Result<()> {
3077    validate_configuration(configuration, explicit)?;
3078    if identity.is_empty() {
3079        return Err(invalid_configuration(
3080            "dataset_identity",
3081            "must not be empty",
3082        ));
3083    }
3084    if configuration.workers != 0 {
3085        return Err(invalid_configuration(
3086            "workers",
3087            "Task 11 exact checkpoints require serial execution",
3088        ));
3089    }
3090    if !configuration.ordered {
3091        return Err(invalid_configuration(
3092            "in_order",
3093            "exact checkpoints require ordered delivery",
3094        ));
3095    }
3096    Ok(())
3097}
3098
3099fn validate_distributed_rank(identity: &PlanCheckpointIdentity, rank: usize) -> Result<()> {
3100    if let Some(distributed) = identity.distributed
3101        && distributed.rank != rank
3102    {
3103        return Err(invalid_configuration(
3104            "rank",
3105            "loader task rank must equal the distributed sampler rank",
3106        ));
3107    }
3108    Ok(())
3109}
3110
3111fn checkpoint_configuration(
3112    identity: &PlanCheckpointIdentity,
3113    builder: &BuilderConfiguration,
3114    pin_status: PinMemoryStatus,
3115) -> Result<crate::LoaderConfiguration> {
3116    let (replicas, distributed_shuffle, distributed_seed, distributed_drop_last) =
3117        match identity.distributed {
3118            Some(distributed) => (
3119                distributed.replicas,
3120                Some(distributed.shuffle),
3121                Some(distributed.seed),
3122                Some(distributed.drop_last),
3123            ),
3124            None => (1, None, None, None),
3125        };
3126    let pin_request = checkpoint_pin_request(builder.pin_memory)?;
3127    let pin_status = checkpoint_pin_status(pin_status)?;
3128    Ok(crate::LoaderConfiguration {
3129        batch_size: identity.batch_size,
3130        drop_last: identity.drop_last,
3131        workers: builder.workers,
3132        prefetch_factor: builder.prefetch_factor,
3133        in_order: true,
3134        loader_seed: builder.loader_seed,
3135        rank: builder.rank,
3136        replicas,
3137        distributed_shuffle,
3138        distributed_seed,
3139        distributed_drop_last,
3140        sampler_kind: identity.sampler_kind.clone(),
3141        pin_request,
3142        pin_status,
3143    })
3144}
3145
3146fn checkpoint_pin_request(request: PinRequest) -> Result<CheckpointPinRequest> {
3147    Ok(match request {
3148        PinRequest::Disabled => CheckpointPinRequest::Disabled,
3149        PinRequest::Auto => CheckpointPinRequest::Auto,
3150        PinRequest::Explicit(Device::Cuda(index)) => CheckpointPinRequest::ExplicitCuda { index },
3151        PinRequest::Explicit(device) => {
3152            return Err(invalid_configuration(
3153                "pin_memory",
3154                format!("unsupported checkpoint pin device {device:?}"),
3155            ));
3156        }
3157    })
3158}
3159
3160fn checkpoint_pin_status(status: PinMemoryStatus) -> Result<CheckpointPinStatus> {
3161    Ok(match status {
3162        PinMemoryStatus::Disabled => CheckpointPinStatus::Disabled,
3163        PinMemoryStatus::DisabledNoAccelerator => CheckpointPinStatus::DisabledNoAccelerator,
3164        PinMemoryStatus::Enabled(Device::Cuda(index)) => CheckpointPinStatus::Cuda { index },
3165        PinMemoryStatus::Enabled(device) => {
3166            return Err(invalid_configuration(
3167                "pin_memory",
3168                format!("unsupported effective checkpoint pin device {device:?}"),
3169            ));
3170        }
3171    })
3172}
3173
3174fn validate_static_loader_state_envelope<D, S, T, C>(
3175    state: &LoaderState<D, S, T, C>,
3176    identity: &str,
3177    builder: &BuilderConfiguration,
3178    pin_status: PinMemoryStatus,
3179    automatic_batching: bool,
3180    expected_generation: u64,
3181) -> Result<()> {
3182    let expected_batch_size = automatic_batching.then_some(builder.batch_size);
3183    let expected_drop_last = automatic_batching && builder.drop_last;
3184    let expected_pin_request = checkpoint_pin_request(builder.pin_memory)?;
3185    let expected_pin_status = checkpoint_pin_status(pin_status)?;
3186
3187    require_checkpoint_equal(
3188        "schema_version",
3189        state.schema_version,
3190        LOADER_STATE_SCHEMA_VERSION,
3191    )?;
3192    require_checkpoint_equal(
3193        "dataset_identity",
3194        state.dataset_identity.as_str(),
3195        identity,
3196    )?;
3197    require_checkpoint_equal(
3198        "iterator_generation",
3199        state.iterator_generation,
3200        expected_generation,
3201    )?;
3202    require_checkpoint_equal(
3203        "rng_derivation_version",
3204        state.rng_derivation_version,
3205        TASK_RNG_DERIVATION_VERSION,
3206    )?;
3207    require_checkpoint_equal(
3208        "worker_seed_derivation_version",
3209        state.worker_seed_derivation_version,
3210        WORKER_SEED_DERIVATION_VERSION,
3211    )?;
3212    require_checkpoint_equal(
3213        "batch_size",
3214        state.configuration.batch_size,
3215        expected_batch_size,
3216    )?;
3217    require_checkpoint_equal(
3218        "drop_last",
3219        state.configuration.drop_last,
3220        expected_drop_last,
3221    )?;
3222    require_checkpoint_equal("workers", state.configuration.workers, builder.workers)?;
3223    require_checkpoint_equal(
3224        "prefetch_factor",
3225        state.configuration.prefetch_factor,
3226        builder.prefetch_factor,
3227    )?;
3228    require_checkpoint_equal("in_order", state.configuration.in_order, builder.ordered)?;
3229    require_checkpoint_equal(
3230        "loader_seed",
3231        state.configuration.loader_seed,
3232        builder.loader_seed,
3233    )?;
3234    require_checkpoint_equal("rank", state.configuration.rank, builder.rank)?;
3235    require_checkpoint_equal(
3236        "pin_request",
3237        &state.configuration.pin_request,
3238        &expected_pin_request,
3239    )?;
3240    require_checkpoint_equal(
3241        "pin_status",
3242        &state.configuration.pin_status,
3243        &expected_pin_status,
3244    )
3245}
3246
3247fn validate_plan_loader_state_envelope<D, S, T, C>(
3248    state: &LoaderState<D, S, T, C>,
3249    expected: &crate::LoaderConfiguration,
3250) -> Result<()> {
3251    require_checkpoint_equal("replicas", state.configuration.replicas, expected.replicas)?;
3252    require_checkpoint_equal(
3253        "distributed_shuffle",
3254        state.configuration.distributed_shuffle,
3255        expected.distributed_shuffle,
3256    )?;
3257    require_checkpoint_equal(
3258        "distributed_seed",
3259        state.configuration.distributed_seed,
3260        expected.distributed_seed,
3261    )?;
3262    require_checkpoint_equal(
3263        "distributed_drop_last",
3264        state.configuration.distributed_drop_last,
3265        expected.distributed_drop_last,
3266    )?;
3267    require_checkpoint_equal(
3268        "sampler_kind",
3269        state.configuration.sampler_kind.as_str(),
3270        expected.sampler_kind.as_str(),
3271    )
3272}
3273
3274fn require_checkpoint_equal<T>(field: &'static str, actual: T, expected: T) -> Result<()>
3275where
3276    T: PartialEq,
3277{
3278    if actual != expected {
3279        return Err(invalid_configuration(
3280            field,
3281            "checkpoint value does not match the loader",
3282        ));
3283    }
3284    Ok(())
3285}
3286
3287fn resolve_pin_request(request: PinRequest) -> Result<PinMemoryStatus> {
3288    match request {
3289        PinRequest::Disabled => Ok(PinMemoryStatus::Disabled),
3290        PinRequest::Auto => {
3291            let capabilities = available_devices();
3292            if capabilities.cuda && capabilities.cuda_device_count > 0 {
3293                Ok(PinMemoryStatus::Enabled(Device::Cuda(0)))
3294            } else {
3295                Ok(PinMemoryStatus::DisabledNoAccelerator)
3296            }
3297        }
3298        PinRequest::Explicit(Device::Cuda(index)) => {
3299            let capabilities = available_devices();
3300            if capabilities.cuda && index < capabilities.cuda_device_count {
3301                Ok(PinMemoryStatus::Enabled(Device::Cuda(index)))
3302            } else {
3303                Err(invalid_configuration(
3304                    "pin_memory",
3305                    format!(
3306                        "CUDA device {index} was requested, but the linked runtime exposes {} available CUDA device(s)",
3307                        capabilities.cuda_device_count
3308                    ),
3309                ))
3310            }
3311        }
3312        PinRequest::Explicit(device) => Err(invalid_configuration(
3313            "pin_memory",
3314            format!("only an available CUDA device can back pinned host memory, got {device:?}"),
3315        )),
3316    }
3317}
3318
3319fn validate_auto_batch_boundary(
3320    exact_len: Option<usize>,
3321    batch_size: NonZeroUsize,
3322    drop_last: bool,
3323    next_batch: u64,
3324    next_logical_sample: u64,
3325) -> Result<()> {
3326    let batch_size = u64::try_from(batch_size.get()).map_err(|_| {
3327        invalid_configuration(
3328            "checkpoint",
3329            "batch size does not fit the checkpoint schema",
3330        )
3331    })?;
3332    let complete = next_logical_sample / batch_size;
3333    let aligned = next_logical_sample.is_multiple_of(batch_size);
3334    let final_short = match exact_len {
3335        Some(length) => {
3336            let length = u64::try_from(length).map_err(|_| {
3337                invalid_configuration(
3338                    "checkpoint",
3339                    "sampler length does not fit the checkpoint schema",
3340                )
3341            })?;
3342            !drop_last
3343                && next_logical_sample == length
3344                && !aligned
3345                && next_batch == complete.saturating_add(1)
3346        }
3347        None => false,
3348    };
3349    if (!aligned || next_batch != complete) && !final_short {
3350        return Err(invalid_configuration(
3351            "checkpoint",
3352            "batch and logical sample cursors are not on a visible batch boundary",
3353        ));
3354    }
3355    if let Some(length) = exact_len {
3356        let length = u64::try_from(length).map_err(|_| {
3357            invalid_configuration(
3358                "checkpoint",
3359                "sampler length does not fit the checkpoint schema",
3360            )
3361        })?;
3362        if next_logical_sample > length {
3363            return Err(invalid_configuration(
3364                "checkpoint",
3365                "logical sample cursor exceeds the sampler length",
3366            ));
3367        }
3368    }
3369    Ok(())
3370}
3371
3372fn batch_count(length: usize, batch_size: NonZeroUsize, drop_last: bool) -> usize {
3373    let batch_size = batch_size.get();
3374    let complete = length / batch_size;
3375    complete + usize::from(!drop_last && !length.is_multiple_of(batch_size))
3376}
3377
3378fn invalid_configuration(field: &'static str, reason: impl Into<String>) -> RustTorchError {
3379    RustTorchError::InvalidConfiguration {
3380        field,
3381        reason: reason.into(),
3382    }
3383}