1use std::{
2 collections::BTreeMap,
3 convert::Infallible,
4 marker::PhantomData,
5 num::NonZeroUsize,
6 panic::{AssertUnwindSafe, catch_unwind},
7 sync::Arc,
8 time::Duration,
9};
10
11use rusttorch_core::{Device, Result, RustTorchError, available_devices};
12
13#[path = "loader_checkpoint.rs"]
14mod checkpoint_workers;
15
16use crate::checkpoint::{StatelessTransformFactory, TransactionalTransformFactory};
17use crate::checkpoint::{
18 WorkerCheckpointActive, WorkerCheckpointIteration, WorkerLaneState, WorkerTransformLanes,
19 WorkerTransformState,
20};
21use crate::memory::{ByteBudget, MemoryDisabled, MemoryEnabled, MemoryPolicy};
22use crate::sampler::{
23 BatchSourceCheckpoint, DistributedConfiguration, SamplerCheckpoint, validate_batch_size,
24};
25use crate::worker::{
26 WorkerBatch, WorkerFailure, WorkerMessage, WorkerPool, WorkerPoolConfiguration, WorkerReceive,
27 WorkerRunContext, WorkerSubmit, WorkerTask, validate_worker_pool_capacity,
28};
29use crate::{
30 Auto, BatchSampler, BatchSource, CheckpointActive, CheckpointBuildError, CheckpointDisabled,
31 CheckpointFresh, CheckpointIteration, CheckpointPinRequest, CheckpointPinStatus,
32 CheckpointResume, Checkpointable, CloneTransformFactory, Collate, Dataset, DatasetCheckpoint,
33 Deadline, DefaultCollator, DefaultConverter, Explicit, IdentityTransformFactory,
34 LOADER_STATE_SCHEMA_VERSION, LoaderError, LoaderState, MemoryFootprint, NoWorkerInit,
35 PinDisabled, PinEnabled, PinMemory, PinMemoryStatus, PipelineError, RandomSampler, Sampler,
36 SequentialSampler, TASK_RNG_DERIVATION_VERSION, TaskContext, Transform, TransformFactory,
37 WORKER_SEED_DERIVATION_VERSION, WorkerCheckpoint, WorkerInit,
38};
39
40#[doc(hidden)]
42pub struct BuilderDatasetMarker {
43 _private: (),
44}
45
46#[derive(Clone, Copy, Debug, Default)]
48pub struct SerialExecution;
49
50#[derive(Clone, Copy, Debug, Default)]
52pub struct WorkerExecution;
53
54trait SerialBoundaryPolicy {
55 fn begin_next(&mut self);
56 fn commit_visible(&mut self);
57}
58
59trait WorkerBoundaryPolicy<E> {
60 fn begin_next(&mut self) {}
61 fn submitted(&mut self, _sequence: u64, _logical_end: u64) {}
62 fn committed(&mut self, _sequence: u64, _serial_logical_end: u64) {}
63 fn defer_error(&mut self, _sequence: u64, error: E) -> Option<E> {
64 Some(error)
65 }
66 fn take_error(&mut self, _sequence: u64) -> Option<E> {
67 None
68 }
69}
70impl<E> WorkerBoundaryPolicy<E> for CheckpointDisabled {}
71impl<S, E> WorkerBoundaryPolicy<E> for WorkerCheckpointIteration<S, E> {
72 fn begin_next(&mut self) {
73 self.boundary_valid = false;
74 }
75 fn submitted(&mut self, sequence: u64, logical_end: u64) {
76 self.logical_ends.insert(sequence, logical_end);
77 }
78 fn committed(&mut self, sequence: u64, serial_logical_end: u64) {
79 self.next_logical_sample = self
80 .logical_ends
81 .remove(&sequence)
82 .unwrap_or(serial_logical_end);
83 self.boundary_valid = true;
84 if let Some(barrier) = &self.barrier {
85 barrier
86 .committed
87 .store(sequence + 1, std::sync::atomic::Ordering::Release);
88 }
89 }
90 fn defer_error(&mut self, sequence: u64, error: E) -> Option<E> {
91 self.errors.insert(sequence, error);
92 None
93 }
94 fn take_error(&mut self, sequence: u64) -> Option<E> {
95 self.errors.remove(&sequence)
96 }
97}
98
99impl SerialBoundaryPolicy for CheckpointDisabled {
100 fn begin_next(&mut self) {}
101 fn commit_visible(&mut self) {}
102}
103
104impl SerialBoundaryPolicy for CheckpointIteration {
105 fn begin_next(&mut self) {
106 self.boundary_valid = false;
107 }
108
109 fn commit_visible(&mut self) {
110 self.boundary_valid = true;
111 }
112}
113
114type TransformOutput<D, F> =
115 <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<
116 <D as Dataset>::Sample,
117 >>::Output;
118type TransformFailure<D, F> =
119 <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<
120 <D as Dataset>::Sample,
121 >>::Error;
122type PlanFailure<D, P, C, F> = <P as LoaderPlan<TransformOutput<D, F>, C>>::Error;
123type IterPipelineError<D, P, C, F, I> = PipelineError<
124 <D as Dataset>::Error,
125 TransformFailure<D, F>,
126 PlanFailure<D, P, C, F>,
127 <F as TransformFactory<<D as Dataset>::Sample>>::Error,
128 <I as WorkerInit>::Error,
129>;
130type IterError<D, P, C, F, I> = LoaderError<IterPipelineError<D, P, C, F, I>>;
131type IterResult<D, P, C, F, I> = std::result::Result<
132 <P as LoaderPlan<TransformOutput<D, F>, C>>::Batch,
133 IterError<D, P, C, F, I>,
134>;
135type CompletedBatch<D, F, M> = (
136 usize,
137 WorkerBatch<TransformOutput<D, F>, <M as MemoryPolicy>::Permit>,
138);
139type FootprintFn<D, F> = fn(&[TransformOutput<D, F>]) -> usize;
140type WorkerStageFailure<D, F, I> = WorkerFailure<
141 <D as Dataset>::Error,
142 TransformFailure<D, F>,
143 <F as TransformFactory<<D as Dataset>::Sample>>::Error,
144 <I as WorkerInit>::Error,
145>;
146
147#[doc(hidden)]
148pub trait MapPinPolicy<D, P, C, F>
149where
150 D: Dataset,
151 F: TransformFactory<D::Sample>,
152 P: LoaderPlan<TransformOutput<D, F>, C>,
153{
154 fn pin(batch: P::Batch, device: Device) -> Result<P::Batch>;
155}
156
157impl<D, P, C, F> MapPinPolicy<D, P, C, F> for PinDisabled
158where
159 D: Dataset,
160 F: TransformFactory<D::Sample>,
161 P: LoaderPlan<TransformOutput<D, F>, C>,
162{
163 fn pin(batch: P::Batch, _device: Device) -> Result<P::Batch> {
164 Ok(batch)
165 }
166}
167
168impl<D, P, C, F, Q> MapPinPolicy<D, P, C, F> for PinEnabled<Q>
169where
170 D: Dataset,
171 F: TransformFactory<D::Sample>,
172 P: LoaderPlan<TransformOutput<D, F>, C>,
173 P::Batch: PinMemory,
174{
175 fn pin(batch: P::Batch, device: Device) -> Result<P::Batch> {
176 batch.pin_memory(device)
177 }
178}
179
180struct PendingSubmission {
181 worker: usize,
182 task: WorkerTask,
183 next_submission: u64,
184 logical_end: u64,
185}
186
187impl Dataset for BuilderDatasetMarker {
188 type Sample = ();
189 type Error = Infallible;
190
191 fn len(&self) -> usize {
192 0
193 }
194
195 fn get(&self, _index: usize) -> std::result::Result<Self::Sample, Self::Error> {
196 unreachable!("the DataLoader builder marker never loads samples")
197 }
198}
199
200pub struct AutoBatch<S> {
202 sampler: S,
203 batch_size: NonZeroUsize,
204 drop_last: bool,
205}
206
207pub struct ExplicitBatches<B> {
209 batches: B,
210}
211
212pub struct NoBatch<S, V = DefaultConverter> {
214 sampler: S,
215 converter: V,
216}
217
218#[doc(hidden)]
220pub trait LoaderPlan<Sample, C> {
221 type Iter: Iterator<Item = Vec<usize>>;
223 type Batch;
225 type Error;
227
228 const EPOCH_PANIC_STAGE: &'static str = "sampler epoch";
230 const CREATION_PANIC_STAGE: &'static str = "sampler creation";
232 const REFILL_PANIC_STAGE: &'static str = "sampler refill";
234 const FINISH_PANIC_STAGE: &'static str = "collation";
236
237 fn iter(&self) -> Self::Iter;
239 fn exact_len(&self) -> Option<usize>;
241 fn epoch(&self) -> u64;
243 fn set_epoch(&mut self, epoch: u64);
245 fn finish(
247 &mut self,
248 collator: &mut C,
249 samples: Vec<Sample>,
250 ) -> std::result::Result<Self::Batch, Self::Error>;
251}
252
253#[doc(hidden)]
255pub trait LoaderPlanConfiguration {
256 fn apply_batch_options(&mut self, batch_size: NonZeroUsize, drop_last: bool);
258 fn apply_epoch(&mut self, epoch: u64);
260}
261
262#[doc(hidden)]
263#[derive(Clone)]
264pub struct PlanCheckpointIdentity {
265 batch_size: Option<usize>,
266 drop_last: bool,
267 sampler_kind: String,
268 distributed: Option<DistributedConfiguration>,
269}
270
271#[doc(hidden)]
272pub trait CheckpointPlan<Sample, C>: LoaderPlan<Sample, C> {
273 type State;
274 type CoordinatorState;
275
276 const AUTOMATIC_BATCHING: bool = false;
277
278 fn checkpoint_identity(&self) -> PlanCheckpointIdentity;
279 fn checkpoint_identity_from_state(
281 state: &Self::State,
282 batch_size: NonZeroUsize,
283 drop_last: bool,
284 ) -> Result<PlanCheckpointIdentity>;
285 fn checkpoint_state(&self, next_batch: u64, next_logical_sample: u64) -> Result<Self::State>;
286 fn validate_checkpoint_state(
287 &self,
288 state: &Self::State,
289 epoch: u64,
290 next_batch: u64,
291 next_logical_sample: u64,
292 ) -> Result<()>;
293 fn validate_checkpoint_state_with_batch_options(
294 &self,
295 state: &Self::State,
296 epoch: u64,
297 next_batch: u64,
298 next_logical_sample: u64,
299 _batch_size: NonZeroUsize,
300 _drop_last: bool,
301 ) -> Result<()> {
302 self.validate_checkpoint_state(state, epoch, next_batch, next_logical_sample)
303 }
304 fn restore_checkpoint_iter_validated(&mut self, state: &Self::State) -> Self::Iter;
305 fn save_coordinator(&self, collator: &C) -> Self::CoordinatorState;
306 fn validate_coordinator(&self, collator: &C, state: &Self::CoordinatorState) -> Result<()>;
307 fn restore_coordinator_validated(&mut self, collator: &mut C, state: &Self::CoordinatorState);
308}
309
310impl<Sample, S, C> LoaderPlan<Sample, C> for AutoBatch<S>
311where
312 S: Sampler,
313 C: Collate<Sample>,
314{
315 type Iter = BatchSampler<S::Iter>;
316 type Batch = C::Batch;
317 type Error = C::Error;
318
319 const EPOCH_PANIC_STAGE: &'static str = "sampler epoch";
320 const CREATION_PANIC_STAGE: &'static str = "sampler creation";
321 const REFILL_PANIC_STAGE: &'static str = "sampler refill";
322 const FINISH_PANIC_STAGE: &'static str = "collation";
323
324 fn iter(&self) -> Self::Iter {
325 BatchSampler::from_validated(self.sampler.iter(), self.batch_size, self.drop_last)
326 }
327
328 fn exact_len(&self) -> Option<usize> {
329 self.sampler
330 .exact_len()
331 .map(|length| batch_count(length, self.batch_size, self.drop_last))
332 }
333
334 fn epoch(&self) -> u64 {
335 self.sampler.epoch()
336 }
337
338 fn set_epoch(&mut self, epoch: u64) {
339 self.sampler.set_epoch(epoch);
340 }
341
342 fn finish(
343 &mut self,
344 collator: &mut C,
345 samples: Vec<Sample>,
346 ) -> std::result::Result<Self::Batch, Self::Error> {
347 collator.collate(samples)
348 }
349}
350
351impl<S> LoaderPlanConfiguration for AutoBatch<S>
352where
353 S: Sampler,
354{
355 fn apply_batch_options(&mut self, batch_size: NonZeroUsize, drop_last: bool) {
356 self.batch_size = batch_size;
357 self.drop_last = drop_last;
358 }
359
360 fn apply_epoch(&mut self, epoch: u64) {
361 self.sampler.set_epoch(epoch);
362 }
363}
364
365impl<Sample, S, C> CheckpointPlan<Sample, C> for AutoBatch<S>
366where
367 S: SamplerCheckpoint,
368 C: Collate<Sample> + Checkpointable,
369{
370 type State = S::State;
371 type CoordinatorState = C::State;
372
373 const AUTOMATIC_BATCHING: bool = true;
374
375 fn checkpoint_identity(&self) -> PlanCheckpointIdentity {
376 PlanCheckpointIdentity {
377 batch_size: Some(self.batch_size.get()),
378 drop_last: self.drop_last,
379 sampler_kind: self.sampler.kind().to_owned(),
380 distributed: self.sampler.distributed_configuration(),
381 }
382 }
383
384 fn checkpoint_identity_from_state(
385 state: &Self::State,
386 batch_size: NonZeroUsize,
387 drop_last: bool,
388 ) -> Result<PlanCheckpointIdentity> {
389 Ok(PlanCheckpointIdentity {
390 batch_size: Some(batch_size.get()),
391 drop_last,
392 sampler_kind: S::checkpoint_kind_from_state(state),
393 distributed: S::checkpoint_distributed_configuration_from_state(state)?,
394 })
395 }
396
397 fn checkpoint_state(&self, next_batch: u64, next_logical_sample: u64) -> Result<Self::State> {
398 validate_auto_batch_boundary(
399 self.sampler.exact_len(),
400 self.batch_size,
401 self.drop_last,
402 next_batch,
403 next_logical_sample,
404 )?;
405 self.sampler.checkpoint_state(next_logical_sample)
406 }
407
408 fn validate_checkpoint_state(
409 &self,
410 state: &Self::State,
411 epoch: u64,
412 next_batch: u64,
413 next_logical_sample: u64,
414 ) -> Result<()> {
415 <Self as CheckpointPlan<Sample, C>>::validate_checkpoint_state_with_batch_options(
416 self,
417 state,
418 epoch,
419 next_batch,
420 next_logical_sample,
421 self.batch_size,
422 self.drop_last,
423 )
424 }
425
426 fn validate_checkpoint_state_with_batch_options(
427 &self,
428 state: &Self::State,
429 epoch: u64,
430 next_batch: u64,
431 next_logical_sample: u64,
432 batch_size: NonZeroUsize,
433 drop_last: bool,
434 ) -> Result<()> {
435 validate_auto_batch_boundary(
436 self.sampler.exact_len(),
437 batch_size,
438 drop_last,
439 next_batch,
440 next_logical_sample,
441 )?;
442 self.sampler
443 .validate_checkpoint_state(state, epoch, next_logical_sample)
444 }
445
446 fn restore_checkpoint_iter_validated(&mut self, state: &Self::State) -> Self::Iter {
447 let iterator = self.sampler.restore_checkpoint_iter_validated(state);
448 BatchSampler::from_validated(iterator, self.batch_size, self.drop_last)
449 }
450
451 fn save_coordinator(&self, collator: &C) -> Self::CoordinatorState {
452 collator.save_state()
453 }
454
455 fn validate_coordinator(&self, collator: &C, state: &Self::CoordinatorState) -> Result<()> {
456 collator.validate_state(state)
457 }
458
459 fn restore_coordinator_validated(&mut self, collator: &mut C, state: &Self::CoordinatorState) {
460 collator.load_validated(state);
461 }
462}
463
464impl<Sample, B, C> LoaderPlan<Sample, C> for ExplicitBatches<B>
465where
466 B: BatchSource,
467 C: Collate<Sample>,
468{
469 type Iter = B::Iter;
470 type Batch = C::Batch;
471 type Error = C::Error;
472
473 const EPOCH_PANIC_STAGE: &'static str = "batch source epoch";
474 const CREATION_PANIC_STAGE: &'static str = "batch source creation";
475 const REFILL_PANIC_STAGE: &'static str = "batch source refill";
476 const FINISH_PANIC_STAGE: &'static str = "collation";
477
478 fn iter(&self) -> Self::Iter {
479 self.batches.iter()
480 }
481
482 fn exact_len(&self) -> Option<usize> {
483 self.batches.exact_len()
484 }
485
486 fn epoch(&self) -> u64 {
487 self.batches.epoch()
488 }
489
490 fn set_epoch(&mut self, epoch: u64) {
491 self.batches.set_epoch(epoch);
492 }
493
494 fn finish(
495 &mut self,
496 collator: &mut C,
497 samples: Vec<Sample>,
498 ) -> std::result::Result<Self::Batch, Self::Error> {
499 collator.collate(samples)
500 }
501}
502
503impl<B> LoaderPlanConfiguration for ExplicitBatches<B>
504where
505 B: BatchSource,
506{
507 fn apply_batch_options(&mut self, _batch_size: NonZeroUsize, _drop_last: bool) {}
508
509 fn apply_epoch(&mut self, epoch: u64) {
510 self.batches.set_epoch(epoch);
511 }
512}
513
514impl<Sample, B, C> CheckpointPlan<Sample, C> for ExplicitBatches<B>
515where
516 B: BatchSourceCheckpoint,
517 C: Collate<Sample> + Checkpointable,
518{
519 type State = B::State;
520 type CoordinatorState = C::State;
521
522 fn checkpoint_identity(&self) -> PlanCheckpointIdentity {
523 PlanCheckpointIdentity {
524 batch_size: None,
525 drop_last: false,
526 sampler_kind: self.batches.kind(),
527 distributed: self.batches.distributed_configuration(),
528 }
529 }
530
531 fn checkpoint_identity_from_state(
532 state: &Self::State,
533 _batch_size: NonZeroUsize,
534 _drop_last: bool,
535 ) -> Result<PlanCheckpointIdentity> {
536 Ok(PlanCheckpointIdentity {
537 batch_size: None,
538 drop_last: false,
539 sampler_kind: B::checkpoint_kind_from_state(state),
540 distributed: B::checkpoint_distributed_configuration_from_state(state)?,
541 })
542 }
543
544 fn checkpoint_state(&self, next_batch: u64, next_logical_sample: u64) -> Result<Self::State> {
545 self.batches
546 .checkpoint_state(next_batch, next_logical_sample)
547 }
548
549 fn validate_checkpoint_state(
550 &self,
551 state: &Self::State,
552 epoch: u64,
553 next_batch: u64,
554 next_logical_sample: u64,
555 ) -> Result<()> {
556 self.batches
557 .validate_checkpoint_state(state, epoch, next_batch, next_logical_sample)
558 }
559
560 fn restore_checkpoint_iter_validated(&mut self, state: &Self::State) -> Self::Iter {
561 self.batches.restore_checkpoint_iter_validated(state)
562 }
563
564 fn save_coordinator(&self, collator: &C) -> Self::CoordinatorState {
565 collator.save_state()
566 }
567
568 fn validate_coordinator(&self, collator: &C, state: &Self::CoordinatorState) -> Result<()> {
569 collator.validate_state(state)
570 }
571
572 fn restore_coordinator_validated(&mut self, collator: &mut C, state: &Self::CoordinatorState) {
573 collator.load_validated(state);
574 }
575}
576
577fn singleton(index: usize) -> Vec<usize> {
578 vec![index]
579}
580
581impl<Sample, S, V, C> LoaderPlan<Sample, C> for NoBatch<S, V>
582where
583 S: Sampler,
584 V: Collate<Sample>,
585{
586 type Iter = std::iter::Map<S::Iter, fn(usize) -> Vec<usize>>;
587 type Batch = V::Batch;
588 type Error = V::Error;
589
590 const EPOCH_PANIC_STAGE: &'static str = "sampler epoch";
591 const CREATION_PANIC_STAGE: &'static str = "sampler creation";
592 const REFILL_PANIC_STAGE: &'static str = "sampler refill";
593 const FINISH_PANIC_STAGE: &'static str = "conversion";
594
595 fn iter(&self) -> Self::Iter {
596 self.sampler.iter().map(singleton)
597 }
598
599 fn exact_len(&self) -> Option<usize> {
600 self.sampler.exact_len()
601 }
602
603 fn epoch(&self) -> u64 {
604 self.sampler.epoch()
605 }
606
607 fn set_epoch(&mut self, epoch: u64) {
608 self.sampler.set_epoch(epoch);
609 }
610
611 fn finish(
612 &mut self,
613 _collator: &mut C,
614 samples: Vec<Sample>,
615 ) -> std::result::Result<Self::Batch, Self::Error> {
616 self.converter.collate(samples)
617 }
618}
619
620impl<S, V> LoaderPlanConfiguration for NoBatch<S, V>
621where
622 S: Sampler,
623{
624 fn apply_batch_options(&mut self, _batch_size: NonZeroUsize, _drop_last: bool) {}
625
626 fn apply_epoch(&mut self, epoch: u64) {
627 self.sampler.set_epoch(epoch);
628 }
629}
630
631impl<Sample, S, V, C> CheckpointPlan<Sample, C> for NoBatch<S, V>
632where
633 S: SamplerCheckpoint,
634 V: Collate<Sample> + Checkpointable,
635{
636 type State = S::State;
637 type CoordinatorState = V::State;
638
639 fn checkpoint_identity(&self) -> PlanCheckpointIdentity {
640 PlanCheckpointIdentity {
641 batch_size: None,
642 drop_last: false,
643 sampler_kind: self.sampler.kind().to_owned(),
644 distributed: self.sampler.distributed_configuration(),
645 }
646 }
647
648 fn checkpoint_identity_from_state(
649 state: &Self::State,
650 _batch_size: NonZeroUsize,
651 _drop_last: bool,
652 ) -> Result<PlanCheckpointIdentity> {
653 Ok(PlanCheckpointIdentity {
654 batch_size: None,
655 drop_last: false,
656 sampler_kind: S::checkpoint_kind_from_state(state),
657 distributed: S::checkpoint_distributed_configuration_from_state(state)?,
658 })
659 }
660
661 fn checkpoint_state(&self, next_batch: u64, next_logical_sample: u64) -> Result<Self::State> {
662 if next_batch != next_logical_sample {
663 return Err(invalid_configuration(
664 "checkpoint",
665 "no-batch cursor must advance one batch per logical sample",
666 ));
667 }
668 self.sampler.checkpoint_state(next_logical_sample)
669 }
670
671 fn validate_checkpoint_state(
672 &self,
673 state: &Self::State,
674 epoch: u64,
675 next_batch: u64,
676 next_logical_sample: u64,
677 ) -> Result<()> {
678 if next_batch != next_logical_sample {
679 return Err(invalid_configuration(
680 "checkpoint",
681 "no-batch cursor must advance one batch per logical sample",
682 ));
683 }
684 self.sampler
685 .validate_checkpoint_state(state, epoch, next_logical_sample)
686 }
687
688 fn restore_checkpoint_iter_validated(&mut self, state: &Self::State) -> Self::Iter {
689 self.sampler
690 .restore_checkpoint_iter_validated(state)
691 .map(singleton as fn(usize) -> Vec<usize>)
692 }
693
694 fn save_coordinator(&self, _collator: &C) -> Self::CoordinatorState {
695 self.converter.save_state()
696 }
697
698 fn validate_coordinator(&self, _collator: &C, state: &Self::CoordinatorState) -> Result<()> {
699 self.converter.validate_state(state)
700 }
701
702 fn restore_coordinator_validated(&mut self, _collator: &mut C, state: &Self::CoordinatorState) {
703 self.converter.load_validated(state);
704 }
705}
706
707#[derive(Clone, Copy, Default)]
708struct ExplicitArguments {
709 batch_size: bool,
710 drop_last: bool,
711 sampler: bool,
712 shuffle: bool,
713 batch_sampler: bool,
714 without_batching: bool,
715 prefetch_factor: bool,
716 prefetch_bytes: bool,
717}
718
719#[derive(Clone, Copy)]
720enum PinRequest {
721 Disabled,
722 Auto,
723 Explicit(Device),
724}
725
726#[derive(Clone, Copy)]
727struct BuilderConfiguration {
728 batch_size: usize,
729 drop_last: bool,
730 workers: usize,
731 persistent_workers: bool,
732 timeout: Option<Duration>,
733 ordered: bool,
734 pin_memory: PinRequest,
735 prefetch_factor: Option<usize>,
736 prefetch_bytes: Option<NonZeroUsize>,
737 loader_seed: u64,
738 epoch: u64,
739 rank: usize,
740}
741
742impl Default for BuilderConfiguration {
743 fn default() -> Self {
744 Self {
745 batch_size: 1,
746 drop_last: false,
747 workers: 0,
748 persistent_workers: false,
749 timeout: None,
750 ordered: true,
751 pin_memory: PinRequest::Disabled,
752 prefetch_factor: None,
753 prefetch_bytes: None,
754 loader_seed: 0,
755 epoch: 0,
756 rank: 0,
757 }
758 }
759}
760
761pub struct DataLoaderBuilder<
763 D,
764 P,
765 C,
766 F = IdentityTransformFactory,
767 I = NoWorkerInit,
768 X = SerialExecution,
769 M = MemoryDisabled,
770 N = PinDisabled,
771 K = CheckpointDisabled,
772> {
773 dataset: D,
774 plan: P,
775 collator: C,
776 transform_factory: F,
777 worker_init: I,
778 configuration: BuilderConfiguration,
779 explicit: ExplicitArguments,
780 checkpoint: K,
781 states: PhantomData<(X, M, N)>,
782}
783
784impl<D> DataLoaderBuilder<D, AutoBatch<SequentialSampler>, DefaultCollator>
785where
786 D: Dataset,
787{
788 pub fn new(dataset: D) -> Self {
790 Self {
791 plan: AutoBatch {
792 sampler: SequentialSampler::new(dataset.len()),
793 batch_size: NonZeroUsize::MIN,
794 drop_last: false,
795 },
796 dataset,
797 collator: DefaultCollator,
798 transform_factory: IdentityTransformFactory,
799 worker_init: NoWorkerInit,
800 configuration: BuilderConfiguration::default(),
801 explicit: ExplicitArguments::default(),
802 checkpoint: CheckpointDisabled,
803 states: PhantomData,
804 }
805 }
806}
807
808impl<D, P, C, F, I, X, M, N, K> DataLoaderBuilder<D, P, C, F, I, X, M, N, K> {
809 fn map<P2, C2>(
810 self,
811 transform: impl FnOnce(P, C) -> (P2, C2),
812 ) -> DataLoaderBuilder<D, P2, C2, F, I, X, M, N, K> {
813 let (plan, collator) = transform(self.plan, self.collator);
814 DataLoaderBuilder {
815 dataset: self.dataset,
816 plan,
817 collator,
818 transform_factory: self.transform_factory,
819 worker_init: self.worker_init,
820 configuration: self.configuration,
821 explicit: self.explicit,
822 checkpoint: self.checkpoint,
823 states: PhantomData,
824 }
825 }
826
827 pub fn batch_size(mut self, batch_size: usize) -> Self {
829 self.configuration.batch_size = batch_size;
830 self.explicit.batch_size = true;
831 self
832 }
833
834 pub fn drop_last(mut self, drop_last: bool) -> Self {
836 self.configuration.drop_last = drop_last;
837 self.explicit.drop_last = true;
838 self
839 }
840
841 pub fn workers(
848 mut self,
849 workers: usize,
850 ) -> DataLoaderBuilder<D, P, C, F, I, WorkerExecution, M, N, K> {
851 self.configuration.workers = workers;
852 DataLoaderBuilder {
853 dataset: self.dataset,
854 plan: self.plan,
855 collator: self.collator,
856 transform_factory: self.transform_factory,
857 worker_init: self.worker_init,
858 configuration: self.configuration,
859 explicit: self.explicit,
860 checkpoint: self.checkpoint,
861 states: PhantomData,
862 }
863 }
864
865 pub fn seed(mut self, seed: u64) -> Self {
867 self.configuration.loader_seed = seed;
868 self
869 }
870
871 pub fn epoch(mut self, epoch: u64) -> Self {
873 self.configuration.epoch = epoch;
874 self
875 }
876
877 pub fn rank(mut self, rank: usize) -> Self {
879 self.configuration.rank = rank;
880 self
881 }
882
883 pub fn transform<T>(
885 self,
886 transform: T,
887 ) -> DataLoaderBuilder<D, P, C, CloneTransformFactory<T>, I, X, M, N, K> {
888 DataLoaderBuilder {
889 dataset: self.dataset,
890 plan: self.plan,
891 collator: self.collator,
892 transform_factory: CloneTransformFactory::new(transform),
893 worker_init: self.worker_init,
894 configuration: self.configuration,
895 explicit: self.explicit,
896 checkpoint: self.checkpoint,
897 states: PhantomData,
898 }
899 }
900
901 pub fn transform_factory<F2>(
903 self,
904 factory: F2,
905 ) -> DataLoaderBuilder<D, P, C, F2, I, X, M, N, K> {
906 DataLoaderBuilder {
907 dataset: self.dataset,
908 plan: self.plan,
909 collator: self.collator,
910 transform_factory: factory,
911 worker_init: self.worker_init,
912 configuration: self.configuration,
913 explicit: self.explicit,
914 checkpoint: self.checkpoint,
915 states: PhantomData,
916 }
917 }
918
919 pub fn checkpoint_stateless(
922 self,
923 ) -> DataLoaderBuilder<D, P, C, StatelessTransformFactory<F>, I, X, M, N, K> {
924 DataLoaderBuilder {
925 dataset: self.dataset,
926 plan: self.plan,
927 collator: self.collator,
928 transform_factory: StatelessTransformFactory(self.transform_factory),
929 worker_init: self.worker_init,
930 configuration: self.configuration,
931 explicit: self.explicit,
932 checkpoint: self.checkpoint,
933 states: PhantomData,
934 }
935 }
936
937 pub fn checkpoint_transactional(
940 self,
941 ) -> DataLoaderBuilder<D, P, C, TransactionalTransformFactory<F>, I, X, M, N, K> {
942 DataLoaderBuilder {
943 dataset: self.dataset,
944 plan: self.plan,
945 collator: self.collator,
946 transform_factory: TransactionalTransformFactory(self.transform_factory),
947 worker_init: self.worker_init,
948 configuration: self.configuration,
949 explicit: self.explicit,
950 checkpoint: self.checkpoint,
951 states: PhantomData,
952 }
953 }
954
955 pub fn worker_init<I2>(self, worker_init: I2) -> DataLoaderBuilder<D, P, C, F, I2, X, M, N, K> {
957 DataLoaderBuilder {
958 dataset: self.dataset,
959 plan: self.plan,
960 collator: self.collator,
961 transform_factory: self.transform_factory,
962 worker_init,
963 configuration: self.configuration,
964 explicit: self.explicit,
965 checkpoint: self.checkpoint,
966 states: PhantomData,
967 }
968 }
969
970 pub fn persistent_workers(mut self, persistent: bool) -> Self {
977 self.configuration.persistent_workers = persistent;
978 self
979 }
980
981 pub fn timeout(mut self, timeout: Duration) -> Self {
990 self.configuration.timeout = (!timeout.is_zero()).then_some(timeout);
991 self
992 }
993
994 pub fn ordered(mut self, ordered: bool) -> Self {
996 self.configuration.ordered = ordered;
997 self
998 }
999
1000 pub fn in_order(self, in_order: bool) -> Self {
1002 self.ordered(in_order)
1003 }
1004
1005 pub fn pin_memory(mut self) -> DataLoaderBuilder<D, P, C, F, I, X, M, PinEnabled<Auto>, K> {
1029 self.configuration.pin_memory = PinRequest::Auto;
1030 DataLoaderBuilder {
1031 dataset: self.dataset,
1032 plan: self.plan,
1033 collator: self.collator,
1034 transform_factory: self.transform_factory,
1035 worker_init: self.worker_init,
1036 configuration: self.configuration,
1037 explicit: self.explicit,
1038 checkpoint: self.checkpoint,
1039 states: PhantomData,
1040 }
1041 }
1042
1043 pub fn pin_memory_for(
1045 mut self,
1046 device: Device,
1047 ) -> DataLoaderBuilder<D, P, C, F, I, X, M, PinEnabled<Explicit>, K> {
1048 self.configuration.pin_memory = PinRequest::Explicit(device);
1049 DataLoaderBuilder {
1050 dataset: self.dataset,
1051 plan: self.plan,
1052 collator: self.collator,
1053 transform_factory: self.transform_factory,
1054 worker_init: self.worker_init,
1055 configuration: self.configuration,
1056 explicit: self.explicit,
1057 checkpoint: self.checkpoint,
1058 states: PhantomData,
1059 }
1060 }
1061
1062 pub fn prefetch_factor(mut self, factor: usize) -> Self {
1071 self.configuration.prefetch_factor = Some(factor);
1072 self.explicit.prefetch_factor = true;
1073 self
1074 }
1075
1076 pub fn prefetch_bytes(
1104 mut self,
1105 limit: NonZeroUsize,
1106 ) -> DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, N, K> {
1107 self.configuration.prefetch_bytes = Some(limit);
1108 self.explicit.prefetch_bytes = true;
1109 DataLoaderBuilder {
1110 dataset: self.dataset,
1111 plan: self.plan,
1112 collator: self.collator,
1113 transform_factory: self.transform_factory,
1114 worker_init: self.worker_init,
1115 configuration: self.configuration,
1116 explicit: self.explicit,
1117 checkpoint: self.checkpoint,
1118 states: PhantomData,
1119 }
1120 }
1121
1122 pub fn batch_sampler<B>(
1124 mut self,
1125 batches: B,
1126 ) -> DataLoaderBuilder<D, ExplicitBatches<B>, C, F, I, X, M, N, K> {
1127 self.explicit.batch_sampler = true;
1128 self.map(|_, collator| (ExplicitBatches { batches }, collator))
1129 }
1130
1131 #[allow(clippy::type_complexity)]
1139 fn build_inner(
1140 mut self,
1141 footprint: Option<FootprintFn<D, F>>,
1142 ) -> Result<OwnedDataLoader<D, P, C, F, I, X, M, N, K>>
1143 where
1144 D: Dataset,
1145 F: TransformFactory<D::Sample>,
1146 P: LoaderPlanConfiguration,
1147 I: WorkerInit,
1148 M: MemoryPolicy,
1149 {
1150 validate_configuration(&self.configuration, self.explicit)?;
1151 let pin_memory_status = resolve_pin_request(self.configuration.pin_memory)?;
1152 let batch_size = validate_batch_size(self.configuration.batch_size)?;
1153 let effective_prefetch = match (
1154 self.configuration.workers,
1155 self.configuration.prefetch_factor,
1156 ) {
1157 (0, _) => None,
1158 (_, Some(factor)) => NonZeroUsize::new(factor),
1159 (_, None) => NonZeroUsize::new(2),
1160 };
1161 let outstanding_capacity = match (self.configuration.workers, effective_prefetch) {
1162 (0, _) => None,
1163 (workers, Some(factor)) => {
1164 Some(workers.checked_mul(factor.get()).ok_or_else(|| {
1165 invalid_configuration(
1166 "prefetch_factor",
1167 "workers multiplied by prefetch_factor exceeds usize",
1168 )
1169 })?)
1170 }
1171 (_, None) => unreachable!("positive workers always have an effective prefetch factor"),
1172 };
1173 if let Some(capacity) = outstanding_capacity {
1174 validate_worker_pool_capacity::<D, F, I, M>(
1175 self.configuration.workers,
1176 effective_prefetch
1177 .expect("positive workers have a prefetch factor")
1178 .get(),
1179 capacity,
1180 )?;
1181 }
1182 self.plan
1183 .apply_batch_options(batch_size, self.configuration.drop_last);
1184 self.plan.apply_epoch(self.configuration.epoch);
1185 Ok(OwnedDataLoader {
1186 dataset: Arc::new(self.dataset),
1187 plan: self.plan,
1188 collator: self.collator,
1189 transform_factory: Arc::new(self.transform_factory),
1190 worker_init: Arc::new(self.worker_init),
1191 workers: self.configuration.workers,
1192 persistent_workers: self.configuration.persistent_workers,
1193 timeout: self.configuration.timeout,
1194 ordered: self.configuration.ordered,
1195 pin_memory_status,
1196 footprint,
1197 effective_prefetch_bytes: self.configuration.prefetch_bytes,
1198 effective_prefetch,
1199 outstanding_capacity,
1200 loader_seed: self.configuration.loader_seed,
1201 rank: self.configuration.rank,
1202 next_generation: 0,
1203 persistent_pool: None,
1204 checkpoint: self.checkpoint,
1205 policies: PhantomData,
1206 })
1207 }
1208}
1209
1210impl<D, P, C, F, I, X, M, N> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointDisabled> {
1211 pub fn dataset_identity(
1216 self,
1217 identity: String,
1218 ) -> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointFresh> {
1219 DataLoaderBuilder {
1220 dataset: self.dataset,
1221 plan: self.plan,
1222 collator: self.collator,
1223 transform_factory: self.transform_factory,
1224 worker_init: self.worker_init,
1225 configuration: self.configuration,
1226 explicit: self.explicit,
1227 checkpoint: CheckpointFresh { identity },
1228 states: PhantomData,
1229 }
1230 }
1231}
1232
1233impl<D, P, C, F, I, X, M, N> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointFresh> {
1234 pub fn resume_from<S>(
1236 self,
1237 state: S,
1238 ) -> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointResume<S>> {
1239 DataLoaderBuilder {
1240 dataset: self.dataset,
1241 plan: self.plan,
1242 collator: self.collator,
1243 transform_factory: self.transform_factory,
1244 worker_init: self.worker_init,
1245 configuration: self.configuration,
1246 explicit: self.explicit,
1247 checkpoint: CheckpointResume {
1248 identity: self.checkpoint.identity,
1249 state,
1250 },
1251 states: PhantomData,
1252 }
1253 }
1254}
1255
1256fn resident_bytes<T: MemoryFootprint>(values: &[T]) -> usize {
1257 let mut total = 0usize;
1258 for value in values {
1259 let Some(next) = total.checked_add(value.resident_bytes()) else {
1260 return usize::MAX;
1261 };
1262 total = next;
1263 }
1264 total
1265}
1266
1267impl<D, P, C, F, I, X>
1268 DataLoaderBuilder<D, P, C, F, I, X, MemoryDisabled, PinDisabled, CheckpointDisabled>
1269{
1270 #[allow(clippy::type_complexity)]
1272 pub fn build(
1273 self,
1274 ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryDisabled, PinDisabled, CheckpointDisabled>>
1275 where
1276 D: Dataset,
1277 F: TransformFactory<D::Sample>,
1278 P: LoaderPlanConfiguration,
1279 I: WorkerInit,
1280 {
1281 self.build_inner(None)
1282 }
1283}
1284
1285impl<D, P, C, F, I, X>
1286 DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, PinDisabled, CheckpointDisabled>
1287{
1288 #[allow(clippy::type_complexity)]
1290 pub fn build(
1291 self,
1292 ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryEnabled, PinDisabled, CheckpointDisabled>>
1293 where
1294 D: Dataset,
1295 F: TransformFactory<D::Sample>,
1296 TransformOutput<D, F>: MemoryFootprint,
1297 P: LoaderPlanConfiguration,
1298 I: WorkerInit,
1299 {
1300 self.build_inner(Some(resident_bytes::<TransformOutput<D, F>>))
1301 }
1302}
1303
1304impl<D, P, C, F, I, X, Q>
1305 DataLoaderBuilder<D, P, C, F, I, X, MemoryDisabled, PinEnabled<Q>, CheckpointDisabled>
1306{
1307 #[allow(clippy::type_complexity)]
1309 pub fn build(
1310 self,
1311 ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryDisabled, PinEnabled<Q>, CheckpointDisabled>>
1312 where
1313 D: Dataset,
1314 F: TransformFactory<D::Sample>,
1315 P: LoaderPlanConfiguration + LoaderPlan<TransformOutput<D, F>, C>,
1316 P::Batch: PinMemory,
1317 I: WorkerInit,
1318 {
1319 self.build_inner(None)
1320 }
1321}
1322
1323impl<D, P, C, F, I, X, Q>
1324 DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, PinEnabled<Q>, CheckpointDisabled>
1325{
1326 #[allow(clippy::type_complexity)]
1328 pub fn build(
1329 self,
1330 ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryEnabled, PinEnabled<Q>, CheckpointDisabled>>
1331 where
1332 D: Dataset,
1333 F: TransformFactory<D::Sample>,
1334 TransformOutput<D, F>: MemoryFootprint,
1335 P: LoaderPlanConfiguration + LoaderPlan<TransformOutput<D, F>, C>,
1336 P::Batch: PinMemory,
1337 I: WorkerInit,
1338 {
1339 self.build_inner(Some(resident_bytes::<TransformOutput<D, F>>))
1340 }
1341}
1342
1343#[allow(private_bounds)]
1344impl<D, P, C, F, I, N>
1345 DataLoaderBuilder<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointFresh>
1346where
1347 D: DatasetCheckpoint,
1348 F: TransformFactory<D::Sample>,
1349 F::Transform: WorkerCheckpoint,
1350 P: LoaderPlanConfiguration
1351 + LoaderPlan<TransformOutput<D, F>, C>
1352 + CheckpointPlan<TransformOutput<D, F>, C>,
1353 I: WorkerInit,
1354 N: MapPinPolicy<D, P, C, F>,
1355{
1356 #[allow(clippy::type_complexity)]
1358 pub fn build(
1359 mut self,
1360 ) -> Result<
1361 OwnedDataLoader<
1362 D,
1363 P,
1364 C,
1365 F,
1366 I,
1367 SerialExecution,
1368 MemoryDisabled,
1369 N,
1370 CheckpointActive<P::Iter, F::Transform>,
1371 >,
1372 > {
1373 validate_exact_builder(
1374 &self.configuration,
1375 self.explicit,
1376 &self.checkpoint.identity,
1377 )?;
1378 let pin_memory_status = resolve_pin_request(self.configuration.pin_memory)?;
1379 let batch_size = validate_batch_size(self.configuration.batch_size)?;
1380 self.plan
1381 .apply_batch_options(batch_size, self.configuration.drop_last);
1382 self.plan.apply_epoch(self.configuration.epoch);
1383 let identity = self.plan.checkpoint_identity();
1384 validate_distributed_rank(&identity, self.configuration.rank)?;
1385 let checkpoint_configuration =
1386 checkpoint_configuration(&identity, &self.configuration, pin_memory_status)?;
1387
1388 Ok(OwnedDataLoader {
1389 dataset: Arc::new(self.dataset),
1390 plan: self.plan,
1391 collator: self.collator,
1392 transform_factory: Arc::new(self.transform_factory),
1393 worker_init: Arc::new(self.worker_init),
1394 workers: 0,
1395 persistent_workers: false,
1396 timeout: None,
1397 ordered: true,
1398 pin_memory_status,
1399 footprint: None,
1400 effective_prefetch_bytes: None,
1401 effective_prefetch: None,
1402 outstanding_capacity: None,
1403 loader_seed: self.configuration.loader_seed,
1404 rank: self.configuration.rank,
1405 next_generation: 0,
1406 persistent_pool: None,
1407 checkpoint: CheckpointActive {
1408 identity: self.checkpoint.identity,
1409 configuration: checkpoint_configuration,
1410 pending: None,
1411 },
1412 policies: PhantomData,
1413 })
1414 }
1415}
1416
1417#[allow(private_bounds)]
1418impl<D, P, C, F, I, N, DS, SS, TS, CS>
1419 DataLoaderBuilder<
1420 D,
1421 P,
1422 C,
1423 F,
1424 I,
1425 SerialExecution,
1426 MemoryDisabled,
1427 N,
1428 CheckpointResume<LoaderState<DS, SS, TS, CS>>,
1429 >
1430where
1431 D: DatasetCheckpoint<State = DS>,
1432 F: TransformFactory<D::Sample>,
1433 F::Transform: WorkerCheckpoint<State = TS>,
1434 P: LoaderPlanConfiguration
1435 + LoaderPlan<TransformOutput<D, F>, C>
1436 + CheckpointPlan<TransformOutput<D, F>, C, State = SS, CoordinatorState = CS>,
1437 I: WorkerInit,
1438 N: MapPinPolicy<D, P, C, F>,
1439{
1440 #[allow(clippy::type_complexity)]
1442 pub fn build(
1443 mut self,
1444 ) -> std::result::Result<
1445 OwnedDataLoader<
1446 D,
1447 P,
1448 C,
1449 F,
1450 I,
1451 SerialExecution,
1452 MemoryDisabled,
1453 N,
1454 CheckpointActive<P::Iter, F::Transform>,
1455 >,
1456 CheckpointBuildError<F::Error>,
1457 > {
1458 let CheckpointResume {
1459 identity: requested_identity,
1460 state,
1461 } = self.checkpoint;
1462 validate_exact_builder(&self.configuration, self.explicit, &requested_identity)?;
1463 let pin_memory_status = resolve_pin_request(self.configuration.pin_memory)?;
1464 let batch_size = validate_batch_size(self.configuration.batch_size)?;
1465 validate_static_loader_state_envelope(
1466 &state,
1467 &requested_identity,
1468 &self.configuration,
1469 pin_memory_status,
1470 <P as CheckpointPlan<TransformOutput<D, F>, C>>::AUTOMATIC_BATCHING,
1471 0,
1472 )?;
1473
1474 let plan_identity =
1475 <P as CheckpointPlan<TransformOutput<D, F>, C>>::checkpoint_identity_from_state(
1476 &state.sampler,
1477 batch_size,
1478 self.configuration.drop_last,
1479 )?;
1480 validate_distributed_rank(&plan_identity, self.configuration.rank)?;
1481 let expected_configuration =
1482 checkpoint_configuration(&plan_identity, &self.configuration, pin_memory_status)?;
1483 validate_plan_loader_state_envelope(&state, &expected_configuration)?;
1484
1485 self.dataset.validate_dataset_state(&state.dataset)?;
1486 self.plan.validate_checkpoint_state_with_batch_options(
1487 &state.sampler,
1488 state.epoch,
1489 state.next_batch,
1490 state.next_logical_sample,
1491 batch_size,
1492 self.configuration.drop_last,
1493 )?;
1494 let mut transform = self
1495 .transform_factory
1496 .create(None)
1497 .map_err(CheckpointBuildError::TransformFactory)?;
1498 transform.validate_snapshot(&state.transform)?;
1499 self.plan
1500 .validate_coordinator(&self.collator, &state.collate)?;
1501
1502 self.dataset.restore_dataset_validated(&state.dataset);
1503 self.plan
1504 .apply_batch_options(batch_size, self.configuration.drop_last);
1505 let batches = self.plan.restore_checkpoint_iter_validated(&state.sampler);
1506 transform.restore_validated(&state.transform);
1507 self.plan
1508 .restore_coordinator_validated(&mut self.collator, &state.collate);
1509
1510 Ok(OwnedDataLoader {
1511 dataset: Arc::new(self.dataset),
1512 plan: self.plan,
1513 collator: self.collator,
1514 transform_factory: Arc::new(self.transform_factory),
1515 worker_init: Arc::new(self.worker_init),
1516 workers: 0,
1517 persistent_workers: false,
1518 timeout: None,
1519 ordered: true,
1520 pin_memory_status,
1521 footprint: None,
1522 effective_prefetch_bytes: None,
1523 effective_prefetch: None,
1524 outstanding_capacity: None,
1525 loader_seed: self.configuration.loader_seed,
1526 rank: self.configuration.rank,
1527 next_generation: 0,
1528 persistent_pool: None,
1529 checkpoint: CheckpointActive {
1530 identity: requested_identity,
1531 configuration: expected_configuration,
1532 pending: Some(crate::checkpoint::PendingSerial {
1533 batches,
1534 transform,
1535 next_batch: state.next_batch,
1536 next_logical_sample: state.next_logical_sample,
1537 }),
1538 },
1539 policies: PhantomData,
1540 })
1541 }
1542}
1543
1544impl<D, S, C, F, I, X, M, N, K> DataLoaderBuilder<D, AutoBatch<S>, C, F, I, X, M, N, K>
1545where
1546 D: Dataset,
1547{
1548 pub fn sampler<S2>(
1550 mut self,
1551 sampler: S2,
1552 ) -> DataLoaderBuilder<D, AutoBatch<S2>, C, F, I, X, M, N, K> {
1553 self.explicit.sampler = true;
1554 self.map(|plan, collator| {
1555 (
1556 AutoBatch {
1557 sampler,
1558 batch_size: plan.batch_size,
1559 drop_last: plan.drop_last,
1560 },
1561 collator,
1562 )
1563 })
1564 }
1565
1566 #[allow(clippy::type_complexity)]
1573 pub fn shuffle(
1574 mut self,
1575 seed: u64,
1576 ) -> Result<DataLoaderBuilder<D, AutoBatch<RandomSampler>, C, F, I, X, M, N, K>> {
1577 self.explicit.shuffle = true;
1578 let sampler = RandomSampler::new(self.dataset.len(), seed)?;
1579 Ok(self.map(|plan, collator| {
1580 (
1581 AutoBatch {
1582 sampler,
1583 batch_size: plan.batch_size,
1584 drop_last: plan.drop_last,
1585 },
1586 collator,
1587 )
1588 }))
1589 }
1590
1591 pub fn without_batching(
1593 mut self,
1594 ) -> DataLoaderBuilder<D, NoBatch<S, DefaultConverter>, C, F, I, X, M, N, K> {
1595 self.explicit.without_batching = true;
1596 self.map(|plan, collator| {
1597 (
1598 NoBatch {
1599 sampler: plan.sampler,
1600 converter: DefaultConverter,
1601 },
1602 collator,
1603 )
1604 })
1605 }
1606
1607 pub fn collate<C2>(
1609 self,
1610 collator: C2,
1611 ) -> DataLoaderBuilder<D, AutoBatch<S>, C2, F, I, X, M, N, K> {
1612 self.map(|plan, _| (plan, collator))
1613 }
1614}
1615
1616impl<D, B, C, F, I, X, M, N, K> DataLoaderBuilder<D, ExplicitBatches<B>, C, F, I, X, M, N, K>
1617where
1618 D: Dataset,
1619{
1620 pub fn sampler<S>(
1622 mut self,
1623 sampler: S,
1624 ) -> DataLoaderBuilder<D, AutoBatch<S>, C, F, I, X, M, N, K> {
1625 self.explicit.sampler = true;
1626 let batch_size =
1627 NonZeroUsize::new(self.configuration.batch_size).unwrap_or(NonZeroUsize::MIN);
1628 let drop_last = self.configuration.drop_last;
1629 self.map(|_, collator| {
1630 (
1631 AutoBatch {
1632 sampler,
1633 batch_size,
1634 drop_last,
1635 },
1636 collator,
1637 )
1638 })
1639 }
1640
1641 #[allow(clippy::type_complexity)]
1648 pub fn shuffle(
1649 mut self,
1650 seed: u64,
1651 ) -> Result<DataLoaderBuilder<D, AutoBatch<RandomSampler>, C, F, I, X, M, N, K>> {
1652 self.explicit.shuffle = true;
1653 let sampler = RandomSampler::new(self.dataset.len(), seed)?;
1654 let batch_size =
1655 NonZeroUsize::new(self.configuration.batch_size).unwrap_or(NonZeroUsize::MIN);
1656 let drop_last = self.configuration.drop_last;
1657 Ok(self.map(|_, collator| {
1658 (
1659 AutoBatch {
1660 sampler,
1661 batch_size,
1662 drop_last,
1663 },
1664 collator,
1665 )
1666 }))
1667 }
1668
1669 pub fn without_batching(
1671 mut self,
1672 ) -> DataLoaderBuilder<D, NoBatch<SequentialSampler, DefaultConverter>, C, F, I, X, M, N, K>
1673 {
1674 self.explicit.without_batching = true;
1675 let length = self.dataset.len();
1676 self.map(|_, collator| {
1677 (
1678 NoBatch {
1679 sampler: SequentialSampler::new(length),
1680 converter: DefaultConverter,
1681 },
1682 collator,
1683 )
1684 })
1685 }
1686
1687 pub fn collate<C2>(
1689 self,
1690 collator: C2,
1691 ) -> DataLoaderBuilder<D, ExplicitBatches<B>, C2, F, I, X, M, N, K> {
1692 self.map(|plan, _| (plan, collator))
1693 }
1694}
1695
1696impl<D, S, V, C, F, I, X, M, N, K> DataLoaderBuilder<D, NoBatch<S, V>, C, F, I, X, M, N, K>
1697where
1698 D: Dataset,
1699{
1700 pub fn sampler<S2>(
1702 mut self,
1703 sampler: S2,
1704 ) -> DataLoaderBuilder<D, NoBatch<S2, V>, C, F, I, X, M, N, K> {
1705 self.explicit.sampler = true;
1706 self.map(|plan, collator| {
1707 (
1708 NoBatch {
1709 sampler,
1710 converter: plan.converter,
1711 },
1712 collator,
1713 )
1714 })
1715 }
1716
1717 #[allow(clippy::type_complexity)]
1724 pub fn shuffle(
1725 mut self,
1726 seed: u64,
1727 ) -> Result<DataLoaderBuilder<D, NoBatch<RandomSampler, V>, C, F, I, X, M, N, K>> {
1728 self.explicit.shuffle = true;
1729 let sampler = RandomSampler::new(self.dataset.len(), seed)?;
1730 Ok(self.map(|plan, collator| {
1731 (
1732 NoBatch {
1733 sampler,
1734 converter: plan.converter,
1735 },
1736 collator,
1737 )
1738 }))
1739 }
1740
1741 pub fn convert<V2>(
1743 self,
1744 converter: V2,
1745 ) -> DataLoaderBuilder<D, NoBatch<S, V2>, C, F, I, X, M, N, K> {
1746 self.map(|plan, collator| {
1747 (
1748 NoBatch {
1749 sampler: plan.sampler,
1750 converter,
1751 },
1752 collator,
1753 )
1754 })
1755 }
1756}
1757
1758#[allow(private_bounds)]
1768pub struct OwnedDataLoader<
1769 D,
1770 P,
1771 C,
1772 F = IdentityTransformFactory,
1773 I = NoWorkerInit,
1774 X = SerialExecution,
1775 M = MemoryDisabled,
1776 N = PinDisabled,
1777 K = CheckpointDisabled,
1778> where
1779 D: Dataset,
1780 F: TransformFactory<D::Sample>,
1781 I: WorkerInit,
1782 M: MemoryPolicy,
1783{
1784 dataset: Arc<D>,
1785 plan: P,
1786 collator: C,
1787 transform_factory: Arc<F>,
1788 worker_init: Arc<I>,
1789 workers: usize,
1790 persistent_workers: bool,
1791 timeout: Option<Duration>,
1792 ordered: bool,
1793 pin_memory_status: PinMemoryStatus,
1794 footprint: Option<FootprintFn<D, F>>,
1795 effective_prefetch_bytes: Option<NonZeroUsize>,
1796 effective_prefetch: Option<NonZeroUsize>,
1797 outstanding_capacity: Option<usize>,
1798 loader_seed: u64,
1799 rank: usize,
1800 next_generation: u64,
1801 persistent_pool: Option<WorkerPool<D, F, I, M>>,
1802 checkpoint: K,
1803 policies: PhantomData<(X, M, N)>,
1804}
1805
1806#[allow(private_bounds)]
1807impl<D, P, C, F, I, M, N> OwnedDataLoader<D, P, C, F, I, SerialExecution, M, N, CheckpointDisabled>
1808where
1809 D: Dataset,
1810 F: TransformFactory<D::Sample>,
1811 P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
1812 I: WorkerInit,
1813 M: MemoryPolicy,
1814 N: MapPinPolicy<D, P, C, F>,
1815{
1816 pub fn iter(&mut self) -> LoaderIter<'_, D, P, C, F, I, M, N, CheckpointDisabled> {
1818 let (transform, transform_error) = match self.transform_factory.create(None) {
1819 Ok(transform) => (Some(transform), None),
1820 Err(error) => (None, Some(error)),
1821 };
1822 let batches = transform.as_ref().map(|_| self.plan.iter());
1823 let epoch = self.plan.epoch();
1824 let run_context = WorkerRunContext::new(0, self.loader_seed, epoch);
1825 LoaderIter {
1826 dataset: self.dataset.as_ref(),
1827 plan: &mut self.plan,
1828 collator: &mut self.collator,
1829 batches,
1830 transform,
1831 transform_error,
1832 next_batch: 0,
1833 next_logical_sample: 0,
1834 exhausted: false,
1835 loader_seed: self.loader_seed,
1836 epoch,
1837 rank: self.rank,
1838 run_context,
1839 pin_memory_status: self.pin_memory_status,
1840 checkpoint: CheckpointDisabled,
1841 output: PhantomData,
1842 policies: PhantomData,
1843 }
1844 }
1845}
1846
1847#[allow(private_bounds, private_interfaces)]
1848impl<D, P, C, F, I, N>
1849 OwnedDataLoader<
1850 D,
1851 P,
1852 C,
1853 F,
1854 I,
1855 SerialExecution,
1856 MemoryDisabled,
1857 N,
1858 CheckpointActive<P::Iter, F::Transform>,
1859 >
1860where
1861 D: DatasetCheckpoint,
1862 F: TransformFactory<D::Sample>,
1863 F::Transform: WorkerCheckpoint,
1864 P: LoaderPlan<TransformOutput<D, F>, C> + CheckpointPlan<TransformOutput<D, F>, C>,
1865 I: WorkerInit,
1866 N: MapPinPolicy<D, P, C, F>,
1867{
1868 pub fn iter(
1870 &mut self,
1871 ) -> LoaderIter<'_, D, P, C, F, I, MemoryDisabled, N, CheckpointIteration> {
1872 let pending = self.checkpoint.pending.take();
1873 let (batches, transform, transform_error, next_batch, next_logical_sample) = match pending {
1874 Some(pending) => (
1875 Some(pending.batches),
1876 Some(pending.transform),
1877 None,
1878 pending.next_batch,
1879 pending.next_logical_sample,
1880 ),
1881 None => match self.transform_factory.create(None) {
1882 Ok(transform) => (Some(self.plan.iter()), Some(transform), None, 0, 0),
1883 Err(error) => (None, None, Some(error), 0, 0),
1884 },
1885 };
1886 let epoch = self.plan.epoch();
1887 let run_context = WorkerRunContext::new(0, self.loader_seed, epoch);
1888 LoaderIter {
1889 dataset: self.dataset.as_ref(),
1890 plan: &mut self.plan,
1891 collator: &mut self.collator,
1892 batches,
1893 transform,
1894 transform_error,
1895 next_batch,
1896 next_logical_sample,
1897 exhausted: false,
1898 loader_seed: self.loader_seed,
1899 epoch,
1900 rank: self.rank,
1901 run_context,
1902 pin_memory_status: self.pin_memory_status,
1903 checkpoint: CheckpointIteration {
1904 identity: self.checkpoint.identity.clone(),
1905 configuration: self.checkpoint.configuration.clone(),
1906 boundary_valid: true,
1907 },
1908 output: PhantomData,
1909 policies: PhantomData,
1910 }
1911 }
1912
1913 pub fn set_epoch(&mut self, epoch: u64) {
1919 self.plan.set_epoch(epoch);
1920 self.checkpoint.pending = None;
1921 }
1922}
1923
1924#[allow(private_bounds)]
1925impl<D, P, C, F, I, X, M, N> OwnedDataLoader<D, P, C, F, I, X, M, N, CheckpointDisabled>
1926where
1927 D: Dataset,
1928 F: TransformFactory<D::Sample>,
1929 P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
1930 I: WorkerInit,
1931 M: MemoryPolicy,
1932{
1933 pub fn set_epoch(&mut self, epoch: u64) {
1935 self.plan.set_epoch(epoch);
1936 }
1937}
1938
1939#[allow(private_bounds)]
1940impl<D, P, C, F, I, X, M, N, K> OwnedDataLoader<D, P, C, F, I, X, M, N, K>
1941where
1942 D: Dataset,
1943 F: TransformFactory<D::Sample>,
1944 P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
1945 I: WorkerInit,
1946 M: MemoryPolicy,
1947{
1948 pub fn len(&self) -> Option<usize> {
1950 self.plan.exact_len()
1951 }
1952
1953 pub fn is_empty(&self) -> Option<bool> {
1955 self.len().map(|length| length == 0)
1956 }
1957
1958 pub fn epoch(&self) -> u64 {
1960 self.plan.epoch()
1961 }
1962
1963 pub fn workers(&self) -> usize {
1965 self.workers
1966 }
1967
1968 pub fn effective_prefetch_factor(&self) -> Option<NonZeroUsize> {
1970 self.effective_prefetch
1971 }
1972
1973 pub fn is_ordered(&self) -> bool {
1975 self.ordered
1976 }
1977
1978 pub fn pin_memory_enabled(&self) -> bool {
1980 !matches!(self.pin_memory_status, PinMemoryStatus::Disabled)
1981 }
1982
1983 pub fn pin_memory_status(&self) -> PinMemoryStatus {
1985 self.pin_memory_status
1986 }
1987
1988 pub fn effective_prefetch_bytes(&self) -> Option<NonZeroUsize> {
1990 self.effective_prefetch_bytes
1991 }
1992
1993 pub fn timeout(&self) -> Option<Duration> {
1995 self.timeout
1996 }
1997
1998 pub fn persistent_workers_enabled(&self) -> bool {
2000 self.persistent_workers
2001 }
2002}
2003
2004pub struct LoaderIter<
2006 'a,
2007 D,
2008 P,
2009 C,
2010 F = IdentityTransformFactory,
2011 I = NoWorkerInit,
2012 M = MemoryDisabled,
2013 N = PinDisabled,
2014 K = CheckpointDisabled,
2015> where
2016 D: Dataset,
2017 F: TransformFactory<D::Sample>,
2018 P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2019 I: WorkerInit,
2020{
2021 dataset: &'a D,
2022 plan: &'a mut P,
2023 collator: &'a mut C,
2024 batches: Option<P::Iter>,
2025 transform: Option<F::Transform>,
2026 transform_error: Option<F::Error>,
2027 next_batch: u64,
2028 next_logical_sample: u64,
2029 exhausted: bool,
2030 loader_seed: u64,
2031 epoch: u64,
2032 rank: usize,
2033 run_context: WorkerRunContext,
2034 pin_memory_status: PinMemoryStatus,
2035 checkpoint: K,
2036 output: PhantomData<fn() -> I>,
2037 policies: PhantomData<(M, N)>,
2038}
2039
2040#[allow(private_bounds, private_interfaces)]
2041impl<D, P, C, F, I, N> LoaderIter<'_, D, P, C, F, I, MemoryDisabled, N, CheckpointIteration>
2042where
2043 D: DatasetCheckpoint,
2044 F: TransformFactory<D::Sample>,
2045 F::Transform: WorkerCheckpoint,
2046 P: LoaderPlan<TransformOutput<D, F>, C> + CheckpointPlan<TransformOutput<D, F>, C>,
2047 I: WorkerInit,
2048 N: MapPinPolicy<D, P, C, F>,
2049{
2050 #[allow(clippy::type_complexity)]
2057 pub fn checkpoint(
2058 &mut self,
2059 ) -> std::result::Result<
2060 LoaderState<
2061 D::State,
2062 P::State,
2063 <F::Transform as WorkerCheckpoint>::State,
2064 P::CoordinatorState,
2065 >,
2066 IterError<D, P, C, F, I>,
2067 > {
2068 if !self.checkpoint.boundary_valid {
2069 return Err(LoaderError::Checkpoint {
2070 reason: "the active iterator is not at a consumer-visible batch boundary"
2071 .to_owned(),
2072 });
2073 }
2074 let transform = self
2075 .transform
2076 .as_ref()
2077 .ok_or_else(|| LoaderError::Checkpoint {
2078 reason: "the serial transform was not constructed".to_owned(),
2079 })?;
2080 let sampler = self
2081 .plan
2082 .checkpoint_state(self.next_batch, self.next_logical_sample)
2083 .map_err(LoaderError::Configuration)?;
2084 Ok(LoaderState {
2085 schema_version: LOADER_STATE_SCHEMA_VERSION,
2086 dataset_identity: self.checkpoint.identity.clone(),
2087 epoch: self.epoch,
2088 iterator_generation: 0,
2089 next_batch: self.next_batch,
2090 next_logical_sample: self.next_logical_sample,
2091 dataset: self.dataset.snapshot_dataset(),
2092 sampler,
2093 transform: transform.snapshot(),
2094 collate: self.plan.save_coordinator(self.collator),
2095 rng_derivation_version: TASK_RNG_DERIVATION_VERSION,
2096 worker_seed_derivation_version: WORKER_SEED_DERIVATION_VERSION,
2097 configuration: self.checkpoint.configuration.clone(),
2098 })
2099 }
2100}
2101
2102impl<D, P, C, F, I, M, N, K> Iterator for LoaderIter<'_, D, P, C, F, I, M, N, K>
2103where
2104 D: Dataset,
2105 F: TransformFactory<D::Sample>,
2106 P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2107 I: WorkerInit,
2108 N: MapPinPolicy<D, P, C, F>,
2109 K: SerialBoundaryPolicy,
2110{
2111 type Item = std::result::Result<
2112 P::Batch,
2113 LoaderError<
2114 PipelineError<
2115 D::Error,
2116 <F::Transform as Transform<D::Sample>>::Error,
2117 P::Error,
2118 F::Error,
2119 I::Error,
2120 >,
2121 >,
2122 >;
2123
2124 fn next(&mut self) -> Option<Self::Item> {
2125 if self.exhausted {
2126 return None;
2127 }
2128 self.checkpoint.begin_next();
2129 if let Some(source) = self.transform_error.take() {
2130 self.exhausted = true;
2131 return Some(Err(LoaderError::Pipeline {
2132 batch: None,
2133 worker: None,
2134 source: PipelineError::TransformInit(source),
2135 }));
2136 }
2137
2138 let indices = match self.batches.as_mut().and_then(Iterator::next) {
2139 Some(indices) => indices,
2140 None => {
2141 self.exhausted = true;
2142 return None;
2143 }
2144 };
2145 let batch = self.next_batch;
2146 let samples = match self.dataset.get_batch(&indices) {
2147 Ok(samples) => samples,
2148 Err(source) => {
2149 self.exhausted = true;
2150 return Some(Err(LoaderError::Pipeline {
2151 batch: Some(batch),
2152 worker: None,
2153 source: PipelineError::Dataset(source),
2154 }));
2155 }
2156 };
2157 if samples.len() != indices.len() {
2158 self.exhausted = true;
2159 return Some(Err(LoaderError::InvalidBatchCardinality {
2160 batch,
2161 worker: None,
2162 expected: indices.len(),
2163 actual: samples.len(),
2164 }));
2165 }
2166 let mut transformed = Vec::with_capacity(samples.len());
2167 for sample in samples {
2168 let context = TaskContext {
2169 loader_seed: self.loader_seed,
2170 epoch: self.epoch,
2171 rank: self.rank,
2172 logical_sample: self.next_logical_sample,
2173 stage: 0,
2174 cancellation: self.run_context.cancellation.clone(),
2175 deadline: self.run_context.deadline.clone(),
2176 };
2177 let Some(transform) = self.transform.as_mut() else {
2178 self.exhausted = true;
2179 return None;
2180 };
2181 match transform.transform(sample, &context) {
2182 Ok(sample) => transformed.push(sample),
2183 Err(source) => {
2184 self.exhausted = true;
2185 return Some(Err(LoaderError::Pipeline {
2186 batch: Some(batch),
2187 worker: None,
2188 source: PipelineError::Transform(source),
2189 }));
2190 }
2191 }
2192 match self.next_logical_sample.checked_add(1) {
2193 Some(next) => self.next_logical_sample = next,
2194 None => self.exhausted = true,
2195 }
2196 }
2197 let result =
2198 self.plan
2199 .finish(self.collator, transformed)
2200 .map_err(|source| LoaderError::Pipeline {
2201 batch: Some(batch),
2202 worker: None,
2203 source: PipelineError::Collate(source),
2204 })
2205 .and_then(|value| match self.pin_memory_status {
2206 PinMemoryStatus::Enabled(device) => N::pin(value, device)
2207 .map_err(|source| LoaderError::PinMemory { batch, source }),
2208 PinMemoryStatus::Disabled | PinMemoryStatus::DisabledNoAccelerator => Ok(value),
2209 });
2210 if result.is_err() {
2211 self.exhausted = true;
2212 } else if let Some(next) = self.next_batch.checked_add(1) {
2213 self.next_batch = next;
2214 self.checkpoint.commit_visible();
2215 } else {
2216 self.exhausted = true;
2217 }
2218 Some(result)
2219 }
2220}
2221
2222#[allow(private_bounds)]
2224pub struct WorkerLoaderIter<
2225 'a,
2226 D,
2227 P,
2228 C,
2229 F = IdentityTransformFactory,
2230 I = NoWorkerInit,
2231 M = MemoryDisabled,
2232 N = PinDisabled,
2233 K = CheckpointDisabled,
2234> where
2235 D: Dataset,
2236 F: TransformFactory<D::Sample>,
2237 P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2238 I: WorkerInit,
2239 M: MemoryPolicy,
2240{
2241 dataset: Arc<D>,
2242 plan: &'a mut P,
2243 collator: &'a mut C,
2244 batches: Option<P::Iter>,
2245 serial_transform: Option<F::Transform>,
2246 serial_transform_error: Option<F::Error>,
2247 pool: Option<IteratorPool<'a, D, F, I, M>>,
2248 completed: BTreeMap<u64, CompletedBatch<D, F, M>>,
2249 pending_error: Option<IterError<D, P, C, F, I>>,
2250 generation: u64,
2251 run_context: Option<WorkerRunContext>,
2252 workers: usize,
2253 capacity: usize,
2254 ordered: bool,
2255 timeout: Option<Duration>,
2256 loader_seed: u64,
2257 epoch: u64,
2258 rank: usize,
2259 next_submission: u64,
2260 next_visible: u64,
2261 next_logical_sample: u64,
2262 pending_submission: Option<PendingSubmission>,
2263 outstanding: usize,
2264 source_exhausted: bool,
2265 submission_closed: bool,
2266 exhausted: bool,
2267 pin_memory_status: PinMemoryStatus,
2268 _checkpoint: K,
2269 policies: PhantomData<(M, N)>,
2270}
2271
2272enum IteratorPool<'a, D, F, I, M>
2273where
2274 D: Dataset,
2275 F: TransformFactory<D::Sample>,
2276 I: WorkerInit,
2277 M: MemoryPolicy,
2278{
2279 Owned(WorkerPool<D, F, I, M>),
2280 Persistent {
2281 pool: &'a mut WorkerPool<D, F, I, M>,
2282 generation: u64,
2283 },
2284}
2285
2286impl<D, F, I, M> IteratorPool<'_, D, F, I, M>
2287where
2288 D: Dataset,
2289 F: TransformFactory<D::Sample>,
2290 I: WorkerInit,
2291 M: MemoryPolicy,
2292{
2293 fn pool(&self) -> &WorkerPool<D, F, I, M> {
2294 match self {
2295 Self::Owned(pool) => pool,
2296 Self::Persistent { pool, .. } => pool,
2297 }
2298 }
2299
2300 fn pool_mut(&mut self) -> &mut WorkerPool<D, F, I, M> {
2301 match self {
2302 Self::Owned(pool) => pool,
2303 Self::Persistent { pool, .. } => pool,
2304 }
2305 }
2306
2307 fn is_persistent(&self) -> bool {
2308 matches!(self, Self::Persistent { .. })
2309 }
2310}
2311
2312impl<D, F, I, M> Drop for IteratorPool<'_, D, F, I, M>
2313where
2314 D: Dataset,
2315 F: TransformFactory<D::Sample>,
2316 I: WorkerInit,
2317 M: MemoryPolicy,
2318{
2319 fn drop(&mut self) {
2320 if let Self::Persistent { pool, generation } = self
2321 && pool.quiesce(*generation).is_err()
2322 {
2323 pool.poison();
2324 }
2325 }
2326}
2327
2328#[allow(private_bounds)]
2329impl<D, P, C, F, I, M, N> OwnedDataLoader<D, P, C, F, I, WorkerExecution, M, N, CheckpointDisabled>
2330where
2331 D: Dataset + Send + Sync + 'static,
2332 D::Sample: Send + 'static,
2333 D::Error: Send + 'static,
2334 F: TransformFactory<D::Sample> + Send + Sync + 'static,
2335 F::Transform: Send + 'static,
2336 <F::Transform as Transform<D::Sample>>::Output: Send + 'static,
2337 <F::Transform as Transform<D::Sample>>::Error: Send + 'static,
2338 F::Error: Send + 'static,
2339 P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2340 I: WorkerInit + Send + Sync + 'static,
2341 I::Error: Send + 'static,
2342 M: MemoryPolicy + Send + 'static,
2343 N: MapPinPolicy<D, P, C, F>,
2344{
2345 pub fn iter(&mut self) -> WorkerLoaderIter<'_, D, P, C, F, I, M, N, CheckpointDisabled> {
2348 let (epoch, epoch_panicked) = if self.workers == 0 {
2349 (self.plan.epoch(), false)
2350 } else {
2351 match catch_unwind(AssertUnwindSafe(|| self.plan.epoch())) {
2352 Ok(epoch) => (epoch, false),
2353 Err(_) => (0, true),
2354 }
2355 };
2356 let mut iterator = WorkerLoaderIter {
2357 dataset: Arc::clone(&self.dataset),
2358 plan: &mut self.plan,
2359 collator: &mut self.collator,
2360 batches: None,
2361 serial_transform: None,
2362 serial_transform_error: None,
2363 pool: None,
2364 completed: BTreeMap::new(),
2365 pending_error: None,
2366 generation: self.next_generation,
2367 run_context: Some(WorkerRunContext::new(
2368 self.next_generation,
2369 self.loader_seed,
2370 epoch,
2371 )),
2372 workers: self.workers,
2373 capacity: self.outstanding_capacity.unwrap_or(0),
2374 ordered: self.ordered,
2375 timeout: self.timeout,
2376 loader_seed: self.loader_seed,
2377 epoch,
2378 rank: self.rank,
2379 next_submission: 0,
2380 next_visible: 0,
2381 next_logical_sample: 0,
2382 pending_submission: None,
2383 outstanding: 0,
2384 source_exhausted: false,
2385 submission_closed: false,
2386 exhausted: false,
2387 pin_memory_status: self.pin_memory_status,
2388 _checkpoint: CheckpointDisabled,
2389 policies: PhantomData,
2390 };
2391
2392 if epoch_panicked {
2393 iterator.pending_error = Some(LoaderError::CoordinatorPanic {
2394 stage: P::EPOCH_PANIC_STAGE,
2395 batch: None,
2396 });
2397 return iterator;
2398 }
2399 if self.workers == 0 {
2400 match self.transform_factory.create(None) {
2401 Ok(transform) => {
2402 iterator.serial_transform = Some(transform);
2403 iterator.batches = Some(iterator.plan.iter());
2404 }
2405 Err(error) => iterator.serial_transform_error = Some(error),
2406 }
2407 return iterator;
2408 }
2409 let Some(next_generation) = self.next_generation.checked_add(1) else {
2410 iterator.pending_error = Some(LoaderError::Configuration(invalid_configuration(
2411 "workers",
2412 "iterator generation overflowed",
2413 )));
2414 return iterator;
2415 };
2416 iterator.batches = match catch_unwind(AssertUnwindSafe(|| iterator.plan.iter())) {
2417 Ok(batches) => Some(batches),
2418 Err(_) => {
2419 iterator.pending_error = Some(LoaderError::CoordinatorPanic {
2420 stage: P::CREATION_PANIC_STAGE,
2421 batch: None,
2422 });
2423 return iterator;
2424 }
2425 };
2426 self.next_generation = next_generation;
2427 let configuration = WorkerPoolConfiguration {
2428 workers: self.workers,
2429 prefetch_factor: self
2430 .effective_prefetch
2431 .expect("positive workers have a validated prefetch factor")
2432 .get(),
2433 result_capacity: iterator.capacity,
2434 seed_generation: iterator.generation,
2435 loader_seed: self.loader_seed,
2436 rank: self.rank,
2437 };
2438 let run_context =
2439 WorkerRunContext::new(iterator.generation, self.loader_seed, iterator.epoch)
2440 .with_byte_budget(
2441 self.effective_prefetch_bytes
2442 .map(|limit| ByteBudget::new(limit.get())),
2443 );
2444 iterator.run_context = Some(run_context.clone());
2445
2446 if self.persistent_workers {
2447 if self.persistent_pool.is_none() {
2448 match WorkerPool::new(
2449 Arc::clone(&self.dataset),
2450 Arc::clone(&self.transform_factory),
2451 Arc::clone(&self.worker_init),
2452 configuration,
2453 self.footprint,
2454 self.ordered,
2455 ) {
2456 Ok(pool) => self.persistent_pool = Some(pool),
2457 Err(error) => {
2458 iterator.pending_error = Some(LoaderError::Configuration(error));
2459 return iterator;
2460 }
2461 }
2462 }
2463 let pool = self
2464 .persistent_pool
2465 .as_mut()
2466 .expect("persistent pool was created");
2467 if pool.is_poisoned() || pool.start_generation(run_context).is_err() {
2468 iterator.pending_error = Some(LoaderError::ChannelClosed { batch: 0 });
2469 return iterator;
2470 }
2471 iterator.pool = Some(IteratorPool::Persistent {
2472 pool,
2473 generation: iterator.generation,
2474 });
2475 } else {
2476 match WorkerPool::new(
2477 Arc::clone(&self.dataset),
2478 Arc::clone(&self.transform_factory),
2479 Arc::clone(&self.worker_init),
2480 configuration,
2481 self.footprint,
2482 self.ordered,
2483 ) {
2484 Ok(mut pool) => {
2485 if pool.start_generation(run_context).is_err() {
2486 iterator.pending_error = Some(LoaderError::ChannelClosed { batch: 0 });
2487 } else {
2488 iterator.pool = Some(IteratorPool::Owned(pool));
2489 }
2490 }
2491 Err(error) => iterator.pending_error = Some(LoaderError::Configuration(error)),
2492 }
2493 }
2494 if iterator.pending_error.is_none() {
2495 iterator.fill_available();
2496 }
2497 iterator
2498 }
2499}
2500
2501#[allow(private_bounds)]
2502impl<D, P, C, F, I, M, N, K> WorkerLoaderIter<'_, D, P, C, F, I, M, N, K>
2503where
2504 D: Dataset + Send + Sync + 'static,
2505 D::Sample: Send + 'static,
2506 D::Error: Send + 'static,
2507 F: TransformFactory<D::Sample> + Send + Sync + 'static,
2508 F::Transform: Send + 'static,
2509 <F::Transform as Transform<D::Sample>>::Output: Send + 'static,
2510 <F::Transform as Transform<D::Sample>>::Error: Send + 'static,
2511 F::Error: Send + 'static,
2512 P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2513 I: WorkerInit + Send + Sync + 'static,
2514 I::Error: Send + 'static,
2515 M: MemoryPolicy + Send + 'static,
2516 N: MapPinPolicy<D, P, C, F>,
2517 K: WorkerBoundaryPolicy<IterError<D, P, C, F, I>>,
2518{
2519 fn fill_available(&mut self) {
2520 while self.outstanding < self.capacity && !self.source_exhausted && !self.submission_closed
2521 {
2522 if !self.submit_one() {
2523 break;
2524 }
2525 }
2526 }
2527
2528 fn submit_one(&mut self) -> bool {
2529 if let Some(pending) = self.pending_submission.take() {
2530 return self.try_submit(pending);
2531 }
2532 let next_batch = catch_unwind(AssertUnwindSafe(|| {
2533 self.batches.as_mut().and_then(Iterator::next)
2534 }));
2535 let Some(indices) = (match next_batch {
2536 Ok(indices) => indices,
2537 Err(_) => {
2538 self.pending_error = Some(LoaderError::CoordinatorPanic {
2539 stage: P::REFILL_PANIC_STAGE,
2540 batch: Some(self.next_submission),
2541 });
2542 self.submission_closed = true;
2543 return false;
2544 }
2545 }) else {
2546 self.source_exhausted = true;
2547 return false;
2548 };
2549 let batch_sequence = self.next_submission;
2550 let Some(next_submission) = batch_sequence.checked_add(1) else {
2551 self.pending_error = Some(LoaderError::Configuration(invalid_configuration(
2552 "sampler",
2553 "batch sequence overflowed",
2554 )));
2555 self.submission_closed = true;
2556 return false;
2557 };
2558 let logical_start = self.next_logical_sample;
2559 let Some(logical_end) = logical_start.checked_add(indices.len() as u64) else {
2560 self.pending_error = Some(LoaderError::Configuration(invalid_configuration(
2561 "sampler",
2562 "logical sample identifier overflowed",
2563 )));
2564 self.submission_closed = true;
2565 return false;
2566 };
2567 let logical_samples = (logical_start..logical_end).collect();
2568 let worker = batch_sequence as usize % self.workers;
2569 self.try_submit(PendingSubmission {
2570 worker,
2571 task: WorkerTask {
2572 generation: self.generation,
2573 batch_sequence,
2574 logical_samples,
2575 indices,
2576 },
2577 next_submission,
2578 logical_end,
2579 })
2580 }
2581
2582 fn try_submit(&mut self, pending: PendingSubmission) -> bool {
2583 let PendingSubmission {
2584 worker,
2585 task,
2586 next_submission,
2587 logical_end,
2588 } = pending;
2589 match self
2590 .pool
2591 .as_ref()
2592 .expect("submissions require an active worker pool")
2593 .pool()
2594 .submit(worker, task)
2595 {
2596 WorkerSubmit::Submitted => {
2597 self._checkpoint.submitted(next_submission - 1, logical_end);
2598 self.next_submission = next_submission;
2599 self.next_logical_sample = logical_end;
2600 self.outstanding += 1;
2601 true
2602 }
2603 WorkerSubmit::Full(task) => {
2604 self.pending_submission = Some(PendingSubmission {
2605 worker,
2606 task,
2607 next_submission,
2608 logical_end,
2609 });
2610 false
2611 }
2612 WorkerSubmit::Closed => {
2613 self.submission_closed = true;
2614 false
2615 }
2616 }
2617 }
2618
2619 fn stop(&mut self) {
2620 self.stop_inner(false);
2621 }
2622
2623 fn stop_poisoned(&mut self) {
2624 self.stop_inner(true);
2625 }
2626
2627 fn stop_inner(&mut self, poisoned: bool) {
2628 self.exhausted = true;
2629 if let Some(run_context) = &self.run_context {
2630 run_context.cancel();
2631 }
2632 if let Some(mut pool) = self.pool.take() {
2633 let should_poison = poisoned
2634 || (pool.is_persistent() && pool.pool_mut().quiesce(self.generation).is_err());
2635 if should_poison {
2636 pool.pool_mut().poison();
2637 }
2638 }
2639 self.completed.clear();
2640 self.pending_submission = None;
2641 }
2642
2643 fn next_serial(&mut self) -> Option<IterResult<D, P, C, F, I>> {
2644 if let Some(source) = self.serial_transform_error.take() {
2645 self.exhausted = true;
2646 return Some(Err(LoaderError::Pipeline {
2647 batch: None,
2648 worker: None,
2649 source: PipelineError::TransformInit(source),
2650 }));
2651 }
2652 let indices = match self.batches.as_mut().and_then(Iterator::next) {
2653 Some(indices) => indices,
2654 None => {
2655 self.exhausted = true;
2656 return None;
2657 }
2658 };
2659 let batch = self.next_visible;
2660 let samples = match self.dataset.get_batch(&indices) {
2661 Ok(samples) => samples,
2662 Err(source) => {
2663 self.exhausted = true;
2664 return Some(Err(LoaderError::Pipeline {
2665 batch: Some(batch),
2666 worker: None,
2667 source: PipelineError::Dataset(source),
2668 }));
2669 }
2670 };
2671 if samples.len() != indices.len() {
2672 self.exhausted = true;
2673 return Some(Err(LoaderError::InvalidBatchCardinality {
2674 batch,
2675 worker: None,
2676 expected: indices.len(),
2677 actual: samples.len(),
2678 }));
2679 }
2680 let mut transformed = Vec::with_capacity(samples.len());
2681 for sample in samples {
2682 let context = TaskContext {
2683 loader_seed: self.loader_seed,
2684 epoch: self.epoch,
2685 rank: self.rank,
2686 logical_sample: self.next_logical_sample,
2687 stage: 0,
2688 cancellation: self
2689 .run_context
2690 .as_ref()
2691 .expect("serial generation context")
2692 .cancellation
2693 .clone(),
2694 deadline: self
2695 .run_context
2696 .as_ref()
2697 .expect("serial generation context")
2698 .deadline
2699 .clone(),
2700 };
2701 let transform = self
2702 .serial_transform
2703 .as_mut()
2704 .expect("serial transform exists after successful construction");
2705 match transform.transform(sample, &context) {
2706 Ok(sample) => transformed.push(sample),
2707 Err(source) => {
2708 self.exhausted = true;
2709 return Some(Err(LoaderError::Pipeline {
2710 batch: Some(batch),
2711 worker: None,
2712 source: PipelineError::Transform(source),
2713 }));
2714 }
2715 }
2716 self.next_logical_sample += 1;
2717 }
2718 let result =
2719 self.plan
2720 .finish(self.collator, transformed)
2721 .map_err(|source| LoaderError::Pipeline {
2722 batch: Some(batch),
2723 worker: None,
2724 source: PipelineError::Collate(source),
2725 })
2726 .and_then(|value| match self.pin_memory_status {
2727 PinMemoryStatus::Enabled(device) => N::pin(value, device)
2728 .map_err(|source| LoaderError::PinMemory { batch, source }),
2729 PinMemoryStatus::Disabled | PinMemoryStatus::DisabledNoAccelerator => Ok(value),
2730 });
2731 if result.is_err() {
2732 self.exhausted = true;
2733 } else {
2734 self._checkpoint
2735 .committed(self.next_visible, self.next_logical_sample);
2736 self.next_visible += 1;
2737 }
2738 Some(result)
2739 }
2740
2741 fn publish(
2742 &mut self,
2743 worker: usize,
2744 batch: WorkerBatch<
2745 <F::Transform as Transform<D::Sample>>::Output,
2746 <M as MemoryPolicy>::Permit,
2747 >,
2748 ) -> IterResult<D, P, C, F, I> {
2749 debug_assert_eq!(batch.generation, self.generation);
2750 self.outstanding -= 1;
2751 let sequence = batch.batch_sequence;
2752 let WorkerBatch {
2753 batch_sequence,
2754 samples,
2755 permit,
2756 ..
2757 } = batch;
2758 drop(permit);
2759 let result = match catch_unwind(AssertUnwindSafe(|| {
2760 self.plan.finish(self.collator, samples)
2761 })) {
2762 Ok(result) => result.map_err(|source| LoaderError::Pipeline {
2763 batch: Some(sequence),
2764 worker: None,
2765 source: PipelineError::Collate(source),
2766 }),
2767 Err(_) => Err(LoaderError::CoordinatorPanic {
2768 stage: P::FINISH_PANIC_STAGE,
2769 batch: Some(sequence),
2770 }),
2771 };
2772 let result = result.and_then(|value| match self.pin_memory_status {
2773 PinMemoryStatus::Enabled(device) => {
2774 N::pin(value, device).map_err(|source| LoaderError::PinMemory {
2775 batch: batch_sequence,
2776 source,
2777 })
2778 }
2779 PinMemoryStatus::Disabled | PinMemoryStatus::DisabledNoAccelerator => Ok(value),
2780 });
2781 if result.is_ok() {
2782 self._checkpoint
2783 .committed(sequence, self.next_logical_sample);
2784 self.next_visible = self.next_visible.saturating_add(1);
2785 self.fill_available();
2786 } else {
2787 self.stop();
2788 }
2789 let _ = worker;
2790 result
2791 }
2792
2793 fn map_failure(
2794 &mut self,
2795 worker: usize,
2796 batch: Option<u64>,
2797 failure: WorkerStageFailure<D, F, I>,
2798 ) -> IterError<D, P, C, F, I> {
2799 match failure {
2800 WorkerFailure::Dataset(source) => LoaderError::Pipeline {
2801 batch,
2802 worker: Some(worker),
2803 source: PipelineError::Dataset(source),
2804 },
2805 WorkerFailure::Transform(source) => LoaderError::Pipeline {
2806 batch,
2807 worker: Some(worker),
2808 source: PipelineError::Transform(source),
2809 },
2810 WorkerFailure::TransformInit(source) => LoaderError::Pipeline {
2811 batch,
2812 worker: Some(worker),
2813 source: PipelineError::TransformInit(source),
2814 },
2815 WorkerFailure::WorkerInit(source) => LoaderError::Pipeline {
2816 batch,
2817 worker: Some(worker),
2818 source: PipelineError::WorkerInit(source),
2819 },
2820 WorkerFailure::InvalidBatchCardinality { expected, actual } => {
2821 LoaderError::InvalidBatchCardinality {
2822 batch: batch.unwrap_or(self.next_visible),
2823 worker: Some(worker),
2824 expected,
2825 actual,
2826 }
2827 }
2828 WorkerFailure::MemoryLimit { limit, actual } => LoaderError::MemoryLimit {
2829 batch,
2830 worker: Some(worker),
2831 sequence: None,
2832 logical_id: None,
2833 limit,
2834 actual,
2835 },
2836 WorkerFailure::Panic => LoaderError::WorkerPanic { worker, batch },
2837 }
2838 }
2839}
2840
2841impl<D, P, C, F, I, M, N, K> Iterator for WorkerLoaderIter<'_, D, P, C, F, I, M, N, K>
2842where
2843 D: Dataset + Send + Sync + 'static,
2844 D::Sample: Send + 'static,
2845 D::Error: Send + 'static,
2846 F: TransformFactory<D::Sample> + Send + Sync + 'static,
2847 F::Transform: Send + 'static,
2848 <F::Transform as Transform<D::Sample>>::Output: Send + 'static,
2849 <F::Transform as Transform<D::Sample>>::Error: Send + 'static,
2850 F::Error: Send + 'static,
2851 P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2852 I: WorkerInit + Send + Sync + 'static,
2853 I::Error: Send + 'static,
2854 M: MemoryPolicy + Send + 'static,
2855 N: MapPinPolicy<D, P, C, F>,
2856 K: WorkerBoundaryPolicy<IterError<D, P, C, F, I>>,
2857{
2858 type Item = std::result::Result<
2859 P::Batch,
2860 LoaderError<
2861 PipelineError<
2862 D::Error,
2863 <F::Transform as Transform<D::Sample>>::Error,
2864 P::Error,
2865 F::Error,
2866 I::Error,
2867 >,
2868 >,
2869 >;
2870
2871 fn next(&mut self) -> Option<Self::Item> {
2872 self._checkpoint.begin_next();
2873 if self.exhausted {
2874 return None;
2875 }
2876 if let Some(error) = self.pending_error.take() {
2877 self.stop();
2878 return Some(Err(error));
2879 }
2880 if self.workers == 0 {
2881 return self.next_serial();
2882 }
2883 let mut deadline_armed = false;
2884 loop {
2885 if let Some(error) = self._checkpoint.take_error(self.next_visible) {
2886 self.stop();
2887 return Some(Err(error));
2888 }
2889 if self.ordered
2890 && let Some((worker, batch)) = self.completed.remove(&self.next_visible)
2891 {
2892 if deadline_armed {
2893 self.run_context
2894 .as_ref()
2895 .expect("worker context")
2896 .deadline
2897 .disarm();
2898 }
2899 return Some(self.publish(worker, batch));
2900 }
2901 if self.outstanding == 0 && self.source_exhausted {
2902 if deadline_armed {
2903 self.run_context
2904 .as_ref()
2905 .expect("worker context")
2906 .deadline
2907 .disarm();
2908 }
2909 self.stop();
2910 return None;
2911 }
2912 let run_context = self
2913 .run_context
2914 .as_ref()
2915 .expect("worker iteration has a generation context")
2916 .clone();
2917 if !deadline_armed && let Some(timeout) = self.timeout {
2918 run_context.deadline.arm(timeout);
2919 deadline_armed = true;
2920 }
2921 let remaining = self.timeout.and_then(|_| run_context.deadline.remaining());
2922 let received = self
2923 .pool
2924 .as_ref()
2925 .expect("worker iteration has an active pool")
2926 .pool()
2927 .receive(&run_context, remaining);
2928 let completion = match received {
2929 WorkerReceive::Completion(completion) => completion,
2930 WorkerReceive::Timeout => {
2931 let batch = self.next_visible;
2932 run_context.cancel();
2933 run_context.deadline.disarm();
2934 self.stop();
2935 return Some(Err(LoaderError::Timeout { batch }));
2936 }
2937 WorkerReceive::Cancelled => {
2938 run_context.deadline.disarm();
2939 self.stop();
2940 return Some(Err(LoaderError::Cancelled));
2941 }
2942 WorkerReceive::Closed => {
2943 let batch = self.next_visible;
2944 run_context.deadline.disarm();
2945 self.stop_poisoned();
2946 return Some(Err(LoaderError::ChannelClosed { batch }));
2947 }
2948 };
2949 if completion.generation != self.generation {
2950 continue;
2951 }
2952 match completion.result {
2953 Err(failure) => {
2954 let poisons_pool = matches!(
2955 &failure,
2956 WorkerFailure::TransformInit(_)
2957 | WorkerFailure::WorkerInit(_)
2958 | WorkerFailure::Panic
2959 );
2960 let error =
2961 self.map_failure(completion.worker, completion.batch_sequence, failure);
2962 if !poisons_pool && let Some(sequence) = completion.batch_sequence {
2963 match self._checkpoint.defer_error(sequence, error) {
2964 None => continue,
2965 Some(error) => {
2966 run_context.deadline.disarm();
2967 self.stop();
2968 return Some(Err(error));
2969 }
2970 }
2971 }
2972 run_context.deadline.disarm();
2973 if poisons_pool {
2974 self.stop_poisoned();
2975 } else {
2976 self.stop();
2977 }
2978 return Some(Err(error));
2979 }
2980 Ok(WorkerMessage::Batch(batch))
2981 if self.ordered && batch.batch_sequence != self.next_visible =>
2982 {
2983 self.completed
2984 .insert(batch.batch_sequence, (completion.worker, batch));
2985 }
2986 Ok(WorkerMessage::Batch(batch)) => {
2987 run_context.deadline.disarm();
2988 return Some(self.publish(completion.worker, batch));
2989 }
2990 Ok(WorkerMessage::Quiesced) => continue,
2991 }
2992 }
2993 }
2994}
2995
2996fn validate_configuration(
2997 configuration: &BuilderConfiguration,
2998 explicit: ExplicitArguments,
2999) -> Result<()> {
3000 if configuration.batch_size == 0 {
3001 return Err(invalid_configuration(
3002 "batch_size",
3003 "must be greater than zero",
3004 ));
3005 }
3006 if explicit.sampler && explicit.shuffle {
3007 return Err(invalid_configuration(
3008 "sampler",
3009 "cannot be combined with shuffle",
3010 ));
3011 }
3012 if explicit.batch_sampler
3013 && (explicit.batch_size
3014 || explicit.shuffle
3015 || explicit.sampler
3016 || explicit.drop_last
3017 || explicit.without_batching)
3018 {
3019 return Err(invalid_configuration(
3020 "batch_sampler",
3021 "cannot be combined with batch_size, shuffle, sampler, drop_last, or no batching",
3022 ));
3023 }
3024 if explicit.without_batching && explicit.drop_last && configuration.drop_last {
3025 return Err(invalid_configuration(
3026 "drop_last",
3027 "cannot be enabled when automatic batching is disabled",
3028 ));
3029 }
3030 if configuration.workers == 0 && configuration.timeout.is_some() {
3031 return Err(invalid_configuration(
3032 "timeout",
3033 "requires a positive worker count",
3034 ));
3035 }
3036 if configuration
3037 .timeout
3038 .is_some_and(|timeout| !Deadline::can_represent(timeout))
3039 {
3040 return Err(invalid_configuration(
3041 "timeout",
3042 "exceeds the platform monotonic clock range",
3043 ));
3044 }
3045 if explicit.prefetch_factor && configuration.workers == 0 {
3046 return Err(invalid_configuration(
3047 "prefetch_factor",
3048 "requires a positive worker count",
3049 ));
3050 }
3051 if explicit.prefetch_bytes && configuration.workers == 0 {
3052 return Err(invalid_configuration(
3053 "prefetch_bytes",
3054 "requires a positive worker count",
3055 ));
3056 }
3057 if configuration.prefetch_factor == Some(0) {
3058 return Err(invalid_configuration(
3059 "prefetch_factor",
3060 "must be greater than zero",
3061 ));
3062 }
3063 if configuration.persistent_workers && configuration.workers == 0 {
3064 return Err(invalid_configuration(
3065 "persistent_workers",
3066 "requires a positive worker count",
3067 ));
3068 }
3069 Ok(())
3070}
3071
3072fn validate_exact_builder(
3073 configuration: &BuilderConfiguration,
3074 explicit: ExplicitArguments,
3075 identity: &str,
3076) -> Result<()> {
3077 validate_configuration(configuration, explicit)?;
3078 if identity.is_empty() {
3079 return Err(invalid_configuration(
3080 "dataset_identity",
3081 "must not be empty",
3082 ));
3083 }
3084 if configuration.workers != 0 {
3085 return Err(invalid_configuration(
3086 "workers",
3087 "Task 11 exact checkpoints require serial execution",
3088 ));
3089 }
3090 if !configuration.ordered {
3091 return Err(invalid_configuration(
3092 "in_order",
3093 "exact checkpoints require ordered delivery",
3094 ));
3095 }
3096 Ok(())
3097}
3098
3099fn validate_distributed_rank(identity: &PlanCheckpointIdentity, rank: usize) -> Result<()> {
3100 if let Some(distributed) = identity.distributed
3101 && distributed.rank != rank
3102 {
3103 return Err(invalid_configuration(
3104 "rank",
3105 "loader task rank must equal the distributed sampler rank",
3106 ));
3107 }
3108 Ok(())
3109}
3110
3111fn checkpoint_configuration(
3112 identity: &PlanCheckpointIdentity,
3113 builder: &BuilderConfiguration,
3114 pin_status: PinMemoryStatus,
3115) -> Result<crate::LoaderConfiguration> {
3116 let (replicas, distributed_shuffle, distributed_seed, distributed_drop_last) =
3117 match identity.distributed {
3118 Some(distributed) => (
3119 distributed.replicas,
3120 Some(distributed.shuffle),
3121 Some(distributed.seed),
3122 Some(distributed.drop_last),
3123 ),
3124 None => (1, None, None, None),
3125 };
3126 let pin_request = checkpoint_pin_request(builder.pin_memory)?;
3127 let pin_status = checkpoint_pin_status(pin_status)?;
3128 Ok(crate::LoaderConfiguration {
3129 batch_size: identity.batch_size,
3130 drop_last: identity.drop_last,
3131 workers: builder.workers,
3132 prefetch_factor: builder.prefetch_factor,
3133 in_order: true,
3134 loader_seed: builder.loader_seed,
3135 rank: builder.rank,
3136 replicas,
3137 distributed_shuffle,
3138 distributed_seed,
3139 distributed_drop_last,
3140 sampler_kind: identity.sampler_kind.clone(),
3141 pin_request,
3142 pin_status,
3143 })
3144}
3145
3146fn checkpoint_pin_request(request: PinRequest) -> Result<CheckpointPinRequest> {
3147 Ok(match request {
3148 PinRequest::Disabled => CheckpointPinRequest::Disabled,
3149 PinRequest::Auto => CheckpointPinRequest::Auto,
3150 PinRequest::Explicit(Device::Cuda(index)) => CheckpointPinRequest::ExplicitCuda { index },
3151 PinRequest::Explicit(device) => {
3152 return Err(invalid_configuration(
3153 "pin_memory",
3154 format!("unsupported checkpoint pin device {device:?}"),
3155 ));
3156 }
3157 })
3158}
3159
3160fn checkpoint_pin_status(status: PinMemoryStatus) -> Result<CheckpointPinStatus> {
3161 Ok(match status {
3162 PinMemoryStatus::Disabled => CheckpointPinStatus::Disabled,
3163 PinMemoryStatus::DisabledNoAccelerator => CheckpointPinStatus::DisabledNoAccelerator,
3164 PinMemoryStatus::Enabled(Device::Cuda(index)) => CheckpointPinStatus::Cuda { index },
3165 PinMemoryStatus::Enabled(device) => {
3166 return Err(invalid_configuration(
3167 "pin_memory",
3168 format!("unsupported effective checkpoint pin device {device:?}"),
3169 ));
3170 }
3171 })
3172}
3173
3174fn validate_static_loader_state_envelope<D, S, T, C>(
3175 state: &LoaderState<D, S, T, C>,
3176 identity: &str,
3177 builder: &BuilderConfiguration,
3178 pin_status: PinMemoryStatus,
3179 automatic_batching: bool,
3180 expected_generation: u64,
3181) -> Result<()> {
3182 let expected_batch_size = automatic_batching.then_some(builder.batch_size);
3183 let expected_drop_last = automatic_batching && builder.drop_last;
3184 let expected_pin_request = checkpoint_pin_request(builder.pin_memory)?;
3185 let expected_pin_status = checkpoint_pin_status(pin_status)?;
3186
3187 require_checkpoint_equal(
3188 "schema_version",
3189 state.schema_version,
3190 LOADER_STATE_SCHEMA_VERSION,
3191 )?;
3192 require_checkpoint_equal(
3193 "dataset_identity",
3194 state.dataset_identity.as_str(),
3195 identity,
3196 )?;
3197 require_checkpoint_equal(
3198 "iterator_generation",
3199 state.iterator_generation,
3200 expected_generation,
3201 )?;
3202 require_checkpoint_equal(
3203 "rng_derivation_version",
3204 state.rng_derivation_version,
3205 TASK_RNG_DERIVATION_VERSION,
3206 )?;
3207 require_checkpoint_equal(
3208 "worker_seed_derivation_version",
3209 state.worker_seed_derivation_version,
3210 WORKER_SEED_DERIVATION_VERSION,
3211 )?;
3212 require_checkpoint_equal(
3213 "batch_size",
3214 state.configuration.batch_size,
3215 expected_batch_size,
3216 )?;
3217 require_checkpoint_equal(
3218 "drop_last",
3219 state.configuration.drop_last,
3220 expected_drop_last,
3221 )?;
3222 require_checkpoint_equal("workers", state.configuration.workers, builder.workers)?;
3223 require_checkpoint_equal(
3224 "prefetch_factor",
3225 state.configuration.prefetch_factor,
3226 builder.prefetch_factor,
3227 )?;
3228 require_checkpoint_equal("in_order", state.configuration.in_order, builder.ordered)?;
3229 require_checkpoint_equal(
3230 "loader_seed",
3231 state.configuration.loader_seed,
3232 builder.loader_seed,
3233 )?;
3234 require_checkpoint_equal("rank", state.configuration.rank, builder.rank)?;
3235 require_checkpoint_equal(
3236 "pin_request",
3237 &state.configuration.pin_request,
3238 &expected_pin_request,
3239 )?;
3240 require_checkpoint_equal(
3241 "pin_status",
3242 &state.configuration.pin_status,
3243 &expected_pin_status,
3244 )
3245}
3246
3247fn validate_plan_loader_state_envelope<D, S, T, C>(
3248 state: &LoaderState<D, S, T, C>,
3249 expected: &crate::LoaderConfiguration,
3250) -> Result<()> {
3251 require_checkpoint_equal("replicas", state.configuration.replicas, expected.replicas)?;
3252 require_checkpoint_equal(
3253 "distributed_shuffle",
3254 state.configuration.distributed_shuffle,
3255 expected.distributed_shuffle,
3256 )?;
3257 require_checkpoint_equal(
3258 "distributed_seed",
3259 state.configuration.distributed_seed,
3260 expected.distributed_seed,
3261 )?;
3262 require_checkpoint_equal(
3263 "distributed_drop_last",
3264 state.configuration.distributed_drop_last,
3265 expected.distributed_drop_last,
3266 )?;
3267 require_checkpoint_equal(
3268 "sampler_kind",
3269 state.configuration.sampler_kind.as_str(),
3270 expected.sampler_kind.as_str(),
3271 )
3272}
3273
3274fn require_checkpoint_equal<T>(field: &'static str, actual: T, expected: T) -> Result<()>
3275where
3276 T: PartialEq,
3277{
3278 if actual != expected {
3279 return Err(invalid_configuration(
3280 field,
3281 "checkpoint value does not match the loader",
3282 ));
3283 }
3284 Ok(())
3285}
3286
3287fn resolve_pin_request(request: PinRequest) -> Result<PinMemoryStatus> {
3288 match request {
3289 PinRequest::Disabled => Ok(PinMemoryStatus::Disabled),
3290 PinRequest::Auto => {
3291 let capabilities = available_devices();
3292 if capabilities.cuda && capabilities.cuda_device_count > 0 {
3293 Ok(PinMemoryStatus::Enabled(Device::Cuda(0)))
3294 } else {
3295 Ok(PinMemoryStatus::DisabledNoAccelerator)
3296 }
3297 }
3298 PinRequest::Explicit(Device::Cuda(index)) => {
3299 let capabilities = available_devices();
3300 if capabilities.cuda && index < capabilities.cuda_device_count {
3301 Ok(PinMemoryStatus::Enabled(Device::Cuda(index)))
3302 } else {
3303 Err(invalid_configuration(
3304 "pin_memory",
3305 format!(
3306 "CUDA device {index} was requested, but the linked runtime exposes {} available CUDA device(s)",
3307 capabilities.cuda_device_count
3308 ),
3309 ))
3310 }
3311 }
3312 PinRequest::Explicit(device) => Err(invalid_configuration(
3313 "pin_memory",
3314 format!("only an available CUDA device can back pinned host memory, got {device:?}"),
3315 )),
3316 }
3317}
3318
3319fn validate_auto_batch_boundary(
3320 exact_len: Option<usize>,
3321 batch_size: NonZeroUsize,
3322 drop_last: bool,
3323 next_batch: u64,
3324 next_logical_sample: u64,
3325) -> Result<()> {
3326 let batch_size = u64::try_from(batch_size.get()).map_err(|_| {
3327 invalid_configuration(
3328 "checkpoint",
3329 "batch size does not fit the checkpoint schema",
3330 )
3331 })?;
3332 let complete = next_logical_sample / batch_size;
3333 let aligned = next_logical_sample.is_multiple_of(batch_size);
3334 let final_short = match exact_len {
3335 Some(length) => {
3336 let length = u64::try_from(length).map_err(|_| {
3337 invalid_configuration(
3338 "checkpoint",
3339 "sampler length does not fit the checkpoint schema",
3340 )
3341 })?;
3342 !drop_last
3343 && next_logical_sample == length
3344 && !aligned
3345 && next_batch == complete.saturating_add(1)
3346 }
3347 None => false,
3348 };
3349 if (!aligned || next_batch != complete) && !final_short {
3350 return Err(invalid_configuration(
3351 "checkpoint",
3352 "batch and logical sample cursors are not on a visible batch boundary",
3353 ));
3354 }
3355 if let Some(length) = exact_len {
3356 let length = u64::try_from(length).map_err(|_| {
3357 invalid_configuration(
3358 "checkpoint",
3359 "sampler length does not fit the checkpoint schema",
3360 )
3361 })?;
3362 if next_logical_sample > length {
3363 return Err(invalid_configuration(
3364 "checkpoint",
3365 "logical sample cursor exceeds the sampler length",
3366 ));
3367 }
3368 }
3369 Ok(())
3370}
3371
3372fn batch_count(length: usize, batch_size: NonZeroUsize, drop_last: bool) -> usize {
3373 let batch_size = batch_size.get();
3374 let complete = length / batch_size;
3375 complete + usize::from(!drop_last && !length.is_multiple_of(batch_size))
3376}
3377
3378fn invalid_configuration(field: &'static str, reason: impl Into<String>) -> RustTorchError {
3379 RustTorchError::InvalidConfiguration {
3380 field,
3381 reason: reason.into(),
3382 }
3383}