1use std::{
2 collections::BTreeMap,
3 convert::Infallible,
4 marker::PhantomData,
5 num::NonZeroUsize,
6 panic::{AssertUnwindSafe, catch_unwind},
7 sync::Arc,
8 time::Duration,
9};
10
11use rusttorch_core::{Device, Result, RustTorchError, available_devices};
12
13#[path = "loader_checkpoint.rs"]
14mod checkpoint_workers;
15
16use crate::checkpoint::{StatelessTransformFactory, TransactionalTransformFactory};
17use crate::checkpoint::{
18 WorkerCheckpointActive, WorkerCheckpointIteration, WorkerLaneState, WorkerTransformLanes,
19 WorkerTransformState,
20};
21use crate::memory::{ByteBudget, MemoryDisabled, MemoryEnabled, MemoryPolicy};
22use crate::sampler::{
23 BatchSourceCheckpoint, DistributedConfiguration, SamplerCheckpoint, validate_batch_size,
24};
25use crate::worker::{
26 WorkerBatch, WorkerFailure, WorkerMessage, WorkerPool, WorkerPoolConfiguration, WorkerReceive,
27 WorkerRunContext, WorkerSubmit, WorkerTask, validate_worker_pool_capacity,
28};
29use crate::{
30 Auto, BatchSampler, BatchSource, CheckpointActive, CheckpointBuildError, CheckpointDisabled,
31 CheckpointFresh, CheckpointIteration, CheckpointPinRequest, CheckpointPinStatus,
32 CheckpointResume, Checkpointable, CloneTransformFactory, Collate, Dataset, DatasetCheckpoint,
33 Deadline, DefaultCollator, DefaultConverter, Explicit, IdentityTransformFactory,
34 LOADER_STATE_SCHEMA_VERSION, LoaderError, LoaderState, MemoryFootprint, NoWorkerInit,
35 PinDisabled, PinEnabled, PinMemory, PinMemoryStatus, PipelineError, RandomSampler, Sampler,
36 SequentialSampler, TASK_RNG_DERIVATION_VERSION, TaskContext, Transform, TransformFactory,
37 WORKER_SEED_DERIVATION_VERSION, WorkerCheckpoint, WorkerInit,
38};
39
40#[doc(hidden)]
42pub struct BuilderDatasetMarker {
43 _private: (),
44}
45
46#[derive(Clone, Copy, Debug, Default)]
48pub struct SerialExecution;
49
50#[derive(Clone, Copy, Debug, Default)]
52pub struct WorkerExecution;
53
54trait SerialBoundaryPolicy {
55 fn begin_next(&mut self);
56 fn commit_visible(&mut self);
57}
58
59trait WorkerBoundaryPolicy<E> {
60 fn begin_next(&mut self) {}
61 fn submitted(&mut self, _sequence: u64, _logical_end: u64) {}
62 fn committed(&mut self, _sequence: u64, _serial_logical_end: u64) {}
63 fn defer_error(&mut self, _sequence: u64, error: E) -> Option<E> {
64 Some(error)
65 }
66 fn take_error(&mut self, _sequence: u64) -> Option<E> {
67 None
68 }
69}
70impl<E> WorkerBoundaryPolicy<E> for CheckpointDisabled {}
71impl<S, E> WorkerBoundaryPolicy<E> for WorkerCheckpointIteration<S, E> {
72 fn begin_next(&mut self) {
73 self.boundary_valid = false;
74 }
75 fn submitted(&mut self, sequence: u64, logical_end: u64) {
76 self.logical_ends.insert(sequence, logical_end);
77 }
78 fn committed(&mut self, sequence: u64, serial_logical_end: u64) {
79 self.next_logical_sample = self
80 .logical_ends
81 .remove(&sequence)
82 .unwrap_or(serial_logical_end);
83 self.boundary_valid = true;
84 if let Some(barrier) = &self.barrier {
85 barrier
86 .committed
87 .store(sequence + 1, std::sync::atomic::Ordering::Release);
88 }
89 }
90 fn defer_error(&mut self, sequence: u64, error: E) -> Option<E> {
91 self.errors.insert(sequence, error);
92 None
93 }
94 fn take_error(&mut self, sequence: u64) -> Option<E> {
95 self.errors.remove(&sequence)
96 }
97}
98
99impl SerialBoundaryPolicy for CheckpointDisabled {
100 fn begin_next(&mut self) {}
101 fn commit_visible(&mut self) {}
102}
103
104impl SerialBoundaryPolicy for CheckpointIteration {
105 fn begin_next(&mut self) {
106 self.boundary_valid = false;
107 }
108
109 fn commit_visible(&mut self) {
110 self.boundary_valid = true;
111 }
112}
113
114type TransformOutput<D, F> =
115 <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<
116 <D as Dataset>::Sample,
117 >>::Output;
118type TransformFailure<D, F> =
119 <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<
120 <D as Dataset>::Sample,
121 >>::Error;
122type PlanFailure<D, P, C, F> = <P as LoaderPlan<TransformOutput<D, F>, C>>::Error;
123type IterPipelineError<D, P, C, F, I> = PipelineError<
124 <D as Dataset>::Error,
125 TransformFailure<D, F>,
126 PlanFailure<D, P, C, F>,
127 <F as TransformFactory<<D as Dataset>::Sample>>::Error,
128 <I as WorkerInit>::Error,
129>;
130type IterError<D, P, C, F, I> = LoaderError<IterPipelineError<D, P, C, F, I>>;
131type IterResult<D, P, C, F, I> = std::result::Result<
132 <P as LoaderPlan<TransformOutput<D, F>, C>>::Batch,
133 IterError<D, P, C, F, I>,
134>;
135type CompletedBatch<D, F, M> = (
136 usize,
137 WorkerBatch<TransformOutput<D, F>, <M as MemoryPolicy>::Permit>,
138);
139type FootprintFn<D, F> = fn(&[TransformOutput<D, F>]) -> usize;
140type WorkerStageFailure<D, F, I> = WorkerFailure<
141 <D as Dataset>::Error,
142 TransformFailure<D, F>,
143 <F as TransformFactory<<D as Dataset>::Sample>>::Error,
144 <I as WorkerInit>::Error,
145>;
146
147#[doc(hidden)]
148pub trait MapPinPolicy<D, P, C, F>
149where
150 D: Dataset,
151 F: TransformFactory<D::Sample>,
152 P: LoaderPlan<TransformOutput<D, F>, C>,
153{
154 fn pin(batch: P::Batch, device: Device) -> Result<P::Batch>;
155}
156
157impl<D, P, C, F> MapPinPolicy<D, P, C, F> for PinDisabled
158where
159 D: Dataset,
160 F: TransformFactory<D::Sample>,
161 P: LoaderPlan<TransformOutput<D, F>, C>,
162{
163 fn pin(batch: P::Batch, _device: Device) -> Result<P::Batch> {
164 Ok(batch)
165 }
166}
167
168impl<D, P, C, F, Q> MapPinPolicy<D, P, C, F> for PinEnabled<Q>
169where
170 D: Dataset,
171 F: TransformFactory<D::Sample>,
172 P: LoaderPlan<TransformOutput<D, F>, C>,
173 P::Batch: PinMemory,
174{
175 fn pin(batch: P::Batch, device: Device) -> Result<P::Batch> {
176 batch.pin_memory(device)
177 }
178}
179
180struct PendingSubmission {
181 worker: usize,
182 task: WorkerTask,
183 next_submission: u64,
184 logical_end: u64,
185}
186
187impl Dataset for BuilderDatasetMarker {
188 type Sample = ();
189 type Error = Infallible;
190
191 fn len(&self) -> usize {
192 0
193 }
194
195 fn get(&self, _index: usize) -> std::result::Result<Self::Sample, Self::Error> {
196 unreachable!("the DataLoader builder marker never loads samples")
197 }
198}
199
200pub struct AutoBatch<S> {
202 sampler: S,
203 batch_size: NonZeroUsize,
204 drop_last: bool,
205}
206
207pub struct ExplicitBatches<B> {
209 batches: B,
210}
211
212pub struct NoBatch<S, V = DefaultConverter> {
214 sampler: S,
215 converter: V,
216}
217
218#[doc(hidden)]
220pub trait LoaderPlan<Sample, C> {
221 type Iter: Iterator<Item = Vec<usize>>;
223 type Batch;
225 type Error;
227
228 const EPOCH_PANIC_STAGE: &'static str = "sampler epoch";
230 const CREATION_PANIC_STAGE: &'static str = "sampler creation";
232 const REFILL_PANIC_STAGE: &'static str = "sampler refill";
234 const FINISH_PANIC_STAGE: &'static str = "collation";
236
237 fn iter(&self) -> Self::Iter;
239 fn exact_len(&self) -> Option<usize>;
241 fn epoch(&self) -> u64;
243 fn set_epoch(&mut self, epoch: u64);
245 fn finish(
247 &mut self,
248 collator: &mut C,
249 samples: Vec<Sample>,
250 ) -> std::result::Result<Self::Batch, Self::Error>;
251}
252
253#[doc(hidden)]
255pub trait LoaderPlanConfiguration {
256 fn apply_batch_options(&mut self, batch_size: NonZeroUsize, drop_last: bool);
258 fn apply_epoch(&mut self, epoch: u64);
260}
261
262#[doc(hidden)]
263#[derive(Clone)]
264pub struct PlanCheckpointIdentity {
265 batch_size: Option<usize>,
266 drop_last: bool,
267 sampler_kind: String,
268 distributed: Option<DistributedConfiguration>,
269}
270
271#[doc(hidden)]
272pub trait CheckpointPlan<Sample, C>: LoaderPlan<Sample, C> {
273 type State;
274 type CoordinatorState;
275
276 const AUTOMATIC_BATCHING: bool = false;
277
278 fn checkpoint_identity(&self) -> PlanCheckpointIdentity;
279 fn checkpoint_identity_from_state(
281 state: &Self::State,
282 batch_size: NonZeroUsize,
283 drop_last: bool,
284 ) -> Result<PlanCheckpointIdentity>;
285 fn checkpoint_state(&self, next_batch: u64, next_logical_sample: u64) -> Result<Self::State>;
286 fn validate_checkpoint_state(
287 &self,
288 state: &Self::State,
289 epoch: u64,
290 next_batch: u64,
291 next_logical_sample: u64,
292 ) -> Result<()>;
293 fn validate_checkpoint_state_with_batch_options(
294 &self,
295 state: &Self::State,
296 epoch: u64,
297 next_batch: u64,
298 next_logical_sample: u64,
299 _batch_size: NonZeroUsize,
300 _drop_last: bool,
301 ) -> Result<()> {
302 self.validate_checkpoint_state(state, epoch, next_batch, next_logical_sample)
303 }
304 fn restore_checkpoint_iter_validated(&mut self, state: &Self::State) -> Self::Iter;
305 fn save_coordinator(&self, collator: &C) -> Self::CoordinatorState;
306 fn validate_coordinator(&self, collator: &C, state: &Self::CoordinatorState) -> Result<()>;
307 fn restore_coordinator_validated(&mut self, collator: &mut C, state: &Self::CoordinatorState);
308}
309
310impl<Sample, S, C> LoaderPlan<Sample, C> for AutoBatch<S>
311where
312 S: Sampler,
313 C: Collate<Sample>,
314{
315 type Iter = BatchSampler<S::Iter>;
316 type Batch = C::Batch;
317 type Error = C::Error;
318
319 const EPOCH_PANIC_STAGE: &'static str = "sampler epoch";
320 const CREATION_PANIC_STAGE: &'static str = "sampler creation";
321 const REFILL_PANIC_STAGE: &'static str = "sampler refill";
322 const FINISH_PANIC_STAGE: &'static str = "collation";
323
324 fn iter(&self) -> Self::Iter {
325 BatchSampler::from_validated(self.sampler.iter(), self.batch_size, self.drop_last)
326 }
327
328 fn exact_len(&self) -> Option<usize> {
329 self.sampler
330 .exact_len()
331 .map(|length| batch_count(length, self.batch_size, self.drop_last))
332 }
333
334 fn epoch(&self) -> u64 {
335 self.sampler.epoch()
336 }
337
338 fn set_epoch(&mut self, epoch: u64) {
339 self.sampler.set_epoch(epoch);
340 }
341
342 fn finish(
343 &mut self,
344 collator: &mut C,
345 samples: Vec<Sample>,
346 ) -> std::result::Result<Self::Batch, Self::Error> {
347 collator.collate(samples)
348 }
349}
350
351impl<S> LoaderPlanConfiguration for AutoBatch<S>
352where
353 S: Sampler,
354{
355 fn apply_batch_options(&mut self, batch_size: NonZeroUsize, drop_last: bool) {
356 self.batch_size = batch_size;
357 self.drop_last = drop_last;
358 }
359
360 fn apply_epoch(&mut self, epoch: u64) {
361 self.sampler.set_epoch(epoch);
362 }
363}
364
365impl<Sample, S, C> CheckpointPlan<Sample, C> for AutoBatch<S>
366where
367 S: SamplerCheckpoint,
368 C: Collate<Sample> + Checkpointable,
369{
370 type State = S::State;
371 type CoordinatorState = C::State;
372
373 const AUTOMATIC_BATCHING: bool = true;
374
375 fn checkpoint_identity(&self) -> PlanCheckpointIdentity {
376 PlanCheckpointIdentity {
377 batch_size: Some(self.batch_size.get()),
378 drop_last: self.drop_last,
379 sampler_kind: self.sampler.kind().to_owned(),
380 distributed: self.sampler.distributed_configuration(),
381 }
382 }
383
384 fn checkpoint_identity_from_state(
385 state: &Self::State,
386 batch_size: NonZeroUsize,
387 drop_last: bool,
388 ) -> Result<PlanCheckpointIdentity> {
389 Ok(PlanCheckpointIdentity {
390 batch_size: Some(batch_size.get()),
391 drop_last,
392 sampler_kind: S::checkpoint_kind_from_state(state),
393 distributed: S::checkpoint_distributed_configuration_from_state(state)?,
394 })
395 }
396
397 fn checkpoint_state(&self, next_batch: u64, next_logical_sample: u64) -> Result<Self::State> {
398 validate_auto_batch_boundary(
399 self.sampler.exact_len(),
400 self.batch_size,
401 self.drop_last,
402 next_batch,
403 next_logical_sample,
404 )?;
405 self.sampler.checkpoint_state(next_logical_sample)
406 }
407
408 fn validate_checkpoint_state(
409 &self,
410 state: &Self::State,
411 epoch: u64,
412 next_batch: u64,
413 next_logical_sample: u64,
414 ) -> Result<()> {
415 <Self as CheckpointPlan<Sample, C>>::validate_checkpoint_state_with_batch_options(
416 self,
417 state,
418 epoch,
419 next_batch,
420 next_logical_sample,
421 self.batch_size,
422 self.drop_last,
423 )
424 }
425
426 fn validate_checkpoint_state_with_batch_options(
427 &self,
428 state: &Self::State,
429 epoch: u64,
430 next_batch: u64,
431 next_logical_sample: u64,
432 batch_size: NonZeroUsize,
433 drop_last: bool,
434 ) -> Result<()> {
435 validate_auto_batch_boundary(
436 self.sampler.exact_len(),
437 batch_size,
438 drop_last,
439 next_batch,
440 next_logical_sample,
441 )?;
442 self.sampler
443 .validate_checkpoint_state(state, epoch, next_logical_sample)
444 }
445
446 fn restore_checkpoint_iter_validated(&mut self, state: &Self::State) -> Self::Iter {
447 let iterator = self.sampler.restore_checkpoint_iter_validated(state);
448 BatchSampler::from_validated(iterator, self.batch_size, self.drop_last)
449 }
450
451 fn save_coordinator(&self, collator: &C) -> Self::CoordinatorState {
452 collator.save_state()
453 }
454
455 fn validate_coordinator(&self, collator: &C, state: &Self::CoordinatorState) -> Result<()> {
456 collator.validate_state(state)
457 }
458
459 fn restore_coordinator_validated(&mut self, collator: &mut C, state: &Self::CoordinatorState) {
460 collator.load_validated(state);
461 }
462}
463
464impl<Sample, B, C> LoaderPlan<Sample, C> for ExplicitBatches<B>
465where
466 B: BatchSource,
467 C: Collate<Sample>,
468{
469 type Iter = B::Iter;
470 type Batch = C::Batch;
471 type Error = C::Error;
472
473 const EPOCH_PANIC_STAGE: &'static str = "batch source epoch";
474 const CREATION_PANIC_STAGE: &'static str = "batch source creation";
475 const REFILL_PANIC_STAGE: &'static str = "batch source refill";
476 const FINISH_PANIC_STAGE: &'static str = "collation";
477
478 fn iter(&self) -> Self::Iter {
479 self.batches.iter()
480 }
481
482 fn exact_len(&self) -> Option<usize> {
483 self.batches.exact_len()
484 }
485
486 fn epoch(&self) -> u64 {
487 self.batches.epoch()
488 }
489
490 fn set_epoch(&mut self, epoch: u64) {
491 self.batches.set_epoch(epoch);
492 }
493
494 fn finish(
495 &mut self,
496 collator: &mut C,
497 samples: Vec<Sample>,
498 ) -> std::result::Result<Self::Batch, Self::Error> {
499 collator.collate(samples)
500 }
501}
502
503impl<B> LoaderPlanConfiguration for ExplicitBatches<B>
504where
505 B: BatchSource,
506{
507 fn apply_batch_options(&mut self, _batch_size: NonZeroUsize, _drop_last: bool) {}
508
509 fn apply_epoch(&mut self, epoch: u64) {
510 self.batches.set_epoch(epoch);
511 }
512}
513
514impl<Sample, B, C> CheckpointPlan<Sample, C> for ExplicitBatches<B>
515where
516 B: BatchSourceCheckpoint,
517 C: Collate<Sample> + Checkpointable,
518{
519 type State = B::State;
520 type CoordinatorState = C::State;
521
522 fn checkpoint_identity(&self) -> PlanCheckpointIdentity {
523 PlanCheckpointIdentity {
524 batch_size: None,
525 drop_last: false,
526 sampler_kind: self.batches.kind(),
527 distributed: self.batches.distributed_configuration(),
528 }
529 }
530
531 fn checkpoint_identity_from_state(
532 state: &Self::State,
533 _batch_size: NonZeroUsize,
534 _drop_last: bool,
535 ) -> Result<PlanCheckpointIdentity> {
536 Ok(PlanCheckpointIdentity {
537 batch_size: None,
538 drop_last: false,
539 sampler_kind: B::checkpoint_kind_from_state(state),
540 distributed: B::checkpoint_distributed_configuration_from_state(state)?,
541 })
542 }
543
544 fn checkpoint_state(&self, next_batch: u64, next_logical_sample: u64) -> Result<Self::State> {
545 self.batches
546 .checkpoint_state(next_batch, next_logical_sample)
547 }
548
549 fn validate_checkpoint_state(
550 &self,
551 state: &Self::State,
552 epoch: u64,
553 next_batch: u64,
554 next_logical_sample: u64,
555 ) -> Result<()> {
556 self.batches
557 .validate_checkpoint_state(state, epoch, next_batch, next_logical_sample)
558 }
559
560 fn restore_checkpoint_iter_validated(&mut self, state: &Self::State) -> Self::Iter {
561 self.batches.restore_checkpoint_iter_validated(state)
562 }
563
564 fn save_coordinator(&self, collator: &C) -> Self::CoordinatorState {
565 collator.save_state()
566 }
567
568 fn validate_coordinator(&self, collator: &C, state: &Self::CoordinatorState) -> Result<()> {
569 collator.validate_state(state)
570 }
571
572 fn restore_coordinator_validated(&mut self, collator: &mut C, state: &Self::CoordinatorState) {
573 collator.load_validated(state);
574 }
575}
576
577fn singleton(index: usize) -> Vec<usize> {
578 vec![index]
579}
580
581impl<Sample, S, V, C> LoaderPlan<Sample, C> for NoBatch<S, V>
582where
583 S: Sampler,
584 V: Collate<Sample>,
585{
586 type Iter = std::iter::Map<S::Iter, fn(usize) -> Vec<usize>>;
587 type Batch = V::Batch;
588 type Error = V::Error;
589
590 const EPOCH_PANIC_STAGE: &'static str = "sampler epoch";
591 const CREATION_PANIC_STAGE: &'static str = "sampler creation";
592 const REFILL_PANIC_STAGE: &'static str = "sampler refill";
593 const FINISH_PANIC_STAGE: &'static str = "conversion";
594
595 fn iter(&self) -> Self::Iter {
596 self.sampler.iter().map(singleton)
597 }
598
599 fn exact_len(&self) -> Option<usize> {
600 self.sampler.exact_len()
601 }
602
603 fn epoch(&self) -> u64 {
604 self.sampler.epoch()
605 }
606
607 fn set_epoch(&mut self, epoch: u64) {
608 self.sampler.set_epoch(epoch);
609 }
610
611 fn finish(
612 &mut self,
613 _collator: &mut C,
614 samples: Vec<Sample>,
615 ) -> std::result::Result<Self::Batch, Self::Error> {
616 self.converter.collate(samples)
617 }
618}
619
620impl<S, V> LoaderPlanConfiguration for NoBatch<S, V>
621where
622 S: Sampler,
623{
624 fn apply_batch_options(&mut self, _batch_size: NonZeroUsize, _drop_last: bool) {}
625
626 fn apply_epoch(&mut self, epoch: u64) {
627 self.sampler.set_epoch(epoch);
628 }
629}
630
631impl<Sample, S, V, C> CheckpointPlan<Sample, C> for NoBatch<S, V>
632where
633 S: SamplerCheckpoint,
634 V: Collate<Sample> + Checkpointable,
635{
636 type State = S::State;
637 type CoordinatorState = V::State;
638
639 fn checkpoint_identity(&self) -> PlanCheckpointIdentity {
640 PlanCheckpointIdentity {
641 batch_size: None,
642 drop_last: false,
643 sampler_kind: self.sampler.kind().to_owned(),
644 distributed: self.sampler.distributed_configuration(),
645 }
646 }
647
648 fn checkpoint_identity_from_state(
649 state: &Self::State,
650 _batch_size: NonZeroUsize,
651 _drop_last: bool,
652 ) -> Result<PlanCheckpointIdentity> {
653 Ok(PlanCheckpointIdentity {
654 batch_size: None,
655 drop_last: false,
656 sampler_kind: S::checkpoint_kind_from_state(state),
657 distributed: S::checkpoint_distributed_configuration_from_state(state)?,
658 })
659 }
660
661 fn checkpoint_state(&self, next_batch: u64, next_logical_sample: u64) -> Result<Self::State> {
662 if next_batch != next_logical_sample {
663 return Err(invalid_configuration(
664 "checkpoint",
665 "no-batch cursor must advance one batch per logical sample",
666 ));
667 }
668 self.sampler.checkpoint_state(next_logical_sample)
669 }
670
671 fn validate_checkpoint_state(
672 &self,
673 state: &Self::State,
674 epoch: u64,
675 next_batch: u64,
676 next_logical_sample: u64,
677 ) -> Result<()> {
678 if next_batch != next_logical_sample {
679 return Err(invalid_configuration(
680 "checkpoint",
681 "no-batch cursor must advance one batch per logical sample",
682 ));
683 }
684 self.sampler
685 .validate_checkpoint_state(state, epoch, next_logical_sample)
686 }
687
688 fn restore_checkpoint_iter_validated(&mut self, state: &Self::State) -> Self::Iter {
689 self.sampler
690 .restore_checkpoint_iter_validated(state)
691 .map(singleton as fn(usize) -> Vec<usize>)
692 }
693
694 fn save_coordinator(&self, _collator: &C) -> Self::CoordinatorState {
695 self.converter.save_state()
696 }
697
698 fn validate_coordinator(&self, _collator: &C, state: &Self::CoordinatorState) -> Result<()> {
699 self.converter.validate_state(state)
700 }
701
702 fn restore_coordinator_validated(&mut self, _collator: &mut C, state: &Self::CoordinatorState) {
703 self.converter.load_validated(state);
704 }
705}
706
707#[derive(Clone, Copy, Default)]
708struct ExplicitArguments {
709 batch_size: bool,
710 drop_last: bool,
711 sampler: bool,
712 shuffle: bool,
713 batch_sampler: bool,
714 without_batching: bool,
715 prefetch_factor: bool,
716 prefetch_bytes: bool,
717}
718
719#[derive(Clone, Copy)]
720enum PinRequest {
721 Disabled,
722 Auto,
723 Explicit(Device),
724}
725
726#[derive(Clone, Copy)]
727struct BuilderConfiguration {
728 batch_size: usize,
729 drop_last: bool,
730 workers: usize,
731 persistent_workers: bool,
732 timeout: Option<Duration>,
733 ordered: bool,
734 pin_memory: PinRequest,
735 prefetch_factor: Option<usize>,
736 prefetch_bytes: Option<NonZeroUsize>,
737 loader_seed: u64,
738 epoch: u64,
739 rank: usize,
740}
741
742impl Default for BuilderConfiguration {
743 fn default() -> Self {
744 Self {
745 batch_size: 1,
746 drop_last: false,
747 workers: 0,
748 persistent_workers: false,
749 timeout: None,
750 ordered: true,
751 pin_memory: PinRequest::Disabled,
752 prefetch_factor: None,
753 prefetch_bytes: None,
754 loader_seed: 0,
755 epoch: 0,
756 rank: 0,
757 }
758 }
759}
760
761pub struct DataLoaderBuilder<
874 D,
875 P,
876 C,
877 F = IdentityTransformFactory,
878 I = NoWorkerInit,
879 X = SerialExecution,
880 M = MemoryDisabled,
881 N = PinDisabled,
882 K = CheckpointDisabled,
883> {
884 dataset: D,
885 plan: P,
886 collator: C,
887 transform_factory: F,
888 worker_init: I,
889 configuration: BuilderConfiguration,
890 explicit: ExplicitArguments,
891 checkpoint: K,
892 states: PhantomData<(X, M, N)>,
893}
894
895impl<D> DataLoaderBuilder<D, AutoBatch<SequentialSampler>, DefaultCollator>
896where
897 D: Dataset,
898{
899 pub fn new(dataset: D) -> Self {
901 Self {
902 plan: AutoBatch {
903 sampler: SequentialSampler::new(dataset.len()),
904 batch_size: NonZeroUsize::MIN,
905 drop_last: false,
906 },
907 dataset,
908 collator: DefaultCollator,
909 transform_factory: IdentityTransformFactory,
910 worker_init: NoWorkerInit,
911 configuration: BuilderConfiguration::default(),
912 explicit: ExplicitArguments::default(),
913 checkpoint: CheckpointDisabled,
914 states: PhantomData,
915 }
916 }
917}
918
919impl<D, P, C, F, I, X, M, N, K> DataLoaderBuilder<D, P, C, F, I, X, M, N, K> {
920 fn map<P2, C2>(
921 self,
922 transform: impl FnOnce(P, C) -> (P2, C2),
923 ) -> DataLoaderBuilder<D, P2, C2, F, I, X, M, N, K> {
924 let (plan, collator) = transform(self.plan, self.collator);
925 DataLoaderBuilder {
926 dataset: self.dataset,
927 plan,
928 collator,
929 transform_factory: self.transform_factory,
930 worker_init: self.worker_init,
931 configuration: self.configuration,
932 explicit: self.explicit,
933 checkpoint: self.checkpoint,
934 states: PhantomData,
935 }
936 }
937
938 pub fn batch_size(mut self, batch_size: usize) -> Self {
944 self.configuration.batch_size = batch_size;
945 self.explicit.batch_size = true;
946 self
947 }
948
949 pub fn drop_last(mut self, drop_last: bool) -> Self {
954 self.configuration.drop_last = drop_last;
955 self.explicit.drop_last = true;
956 self
957 }
958
959 pub fn workers(
967 mut self,
968 workers: usize,
969 ) -> DataLoaderBuilder<D, P, C, F, I, WorkerExecution, M, N, K> {
970 self.configuration.workers = workers;
971 DataLoaderBuilder {
972 dataset: self.dataset,
973 plan: self.plan,
974 collator: self.collator,
975 transform_factory: self.transform_factory,
976 worker_init: self.worker_init,
977 configuration: self.configuration,
978 explicit: self.explicit,
979 checkpoint: self.checkpoint,
980 states: PhantomData,
981 }
982 }
983
984 pub fn seed(mut self, seed: u64) -> Self {
989 self.configuration.loader_seed = seed;
990 self
991 }
992
993 pub fn epoch(mut self, epoch: u64) -> Self {
995 self.configuration.epoch = epoch;
996 self
997 }
998
999 pub fn rank(mut self, rank: usize) -> Self {
1005 self.configuration.rank = rank;
1006 self
1007 }
1008
1009 pub fn transform<T>(
1011 self,
1012 transform: T,
1013 ) -> DataLoaderBuilder<D, P, C, CloneTransformFactory<T>, I, X, M, N, K> {
1014 DataLoaderBuilder {
1015 dataset: self.dataset,
1016 plan: self.plan,
1017 collator: self.collator,
1018 transform_factory: CloneTransformFactory::new(transform),
1019 worker_init: self.worker_init,
1020 configuration: self.configuration,
1021 explicit: self.explicit,
1022 checkpoint: self.checkpoint,
1023 states: PhantomData,
1024 }
1025 }
1026
1027 pub fn transform_factory<F2>(
1029 self,
1030 factory: F2,
1031 ) -> DataLoaderBuilder<D, P, C, F2, I, X, M, N, K> {
1032 DataLoaderBuilder {
1033 dataset: self.dataset,
1034 plan: self.plan,
1035 collator: self.collator,
1036 transform_factory: factory,
1037 worker_init: self.worker_init,
1038 configuration: self.configuration,
1039 explicit: self.explicit,
1040 checkpoint: self.checkpoint,
1041 states: PhantomData,
1042 }
1043 }
1044
1045 pub fn checkpoint_stateless(
1048 self,
1049 ) -> DataLoaderBuilder<D, P, C, StatelessTransformFactory<F>, I, X, M, N, K> {
1050 DataLoaderBuilder {
1051 dataset: self.dataset,
1052 plan: self.plan,
1053 collator: self.collator,
1054 transform_factory: StatelessTransformFactory(self.transform_factory),
1055 worker_init: self.worker_init,
1056 configuration: self.configuration,
1057 explicit: self.explicit,
1058 checkpoint: self.checkpoint,
1059 states: PhantomData,
1060 }
1061 }
1062
1063 pub fn checkpoint_transactional(
1066 self,
1067 ) -> DataLoaderBuilder<D, P, C, TransactionalTransformFactory<F>, I, X, M, N, K> {
1068 DataLoaderBuilder {
1069 dataset: self.dataset,
1070 plan: self.plan,
1071 collator: self.collator,
1072 transform_factory: TransactionalTransformFactory(self.transform_factory),
1073 worker_init: self.worker_init,
1074 configuration: self.configuration,
1075 explicit: self.explicit,
1076 checkpoint: self.checkpoint,
1077 states: PhantomData,
1078 }
1079 }
1080
1081 pub fn worker_init<I2>(self, worker_init: I2) -> DataLoaderBuilder<D, P, C, F, I2, X, M, N, K> {
1083 DataLoaderBuilder {
1084 dataset: self.dataset,
1085 plan: self.plan,
1086 collator: self.collator,
1087 transform_factory: self.transform_factory,
1088 worker_init,
1089 configuration: self.configuration,
1090 explicit: self.explicit,
1091 checkpoint: self.checkpoint,
1092 states: PhantomData,
1093 }
1094 }
1095
1096 pub fn persistent_workers(mut self, persistent: bool) -> Self {
1103 self.configuration.persistent_workers = persistent;
1104 self
1105 }
1106
1107 pub fn timeout(mut self, timeout: Duration) -> Self {
1116 self.configuration.timeout = (!timeout.is_zero()).then_some(timeout);
1117 self
1118 }
1119
1120 pub fn ordered(mut self, ordered: bool) -> Self {
1125 self.configuration.ordered = ordered;
1126 self
1127 }
1128
1129 pub fn in_order(self, in_order: bool) -> Self {
1131 self.ordered(in_order)
1132 }
1133
1134 pub fn pin_memory(mut self) -> DataLoaderBuilder<D, P, C, F, I, X, M, PinEnabled<Auto>, K> {
1163 self.configuration.pin_memory = PinRequest::Auto;
1164 DataLoaderBuilder {
1165 dataset: self.dataset,
1166 plan: self.plan,
1167 collator: self.collator,
1168 transform_factory: self.transform_factory,
1169 worker_init: self.worker_init,
1170 configuration: self.configuration,
1171 explicit: self.explicit,
1172 checkpoint: self.checkpoint,
1173 states: PhantomData,
1174 }
1175 }
1176
1177 pub fn pin_memory_for(
1179 mut self,
1180 device: Device,
1181 ) -> DataLoaderBuilder<D, P, C, F, I, X, M, PinEnabled<Explicit>, K> {
1182 self.configuration.pin_memory = PinRequest::Explicit(device);
1183 DataLoaderBuilder {
1184 dataset: self.dataset,
1185 plan: self.plan,
1186 collator: self.collator,
1187 transform_factory: self.transform_factory,
1188 worker_init: self.worker_init,
1189 configuration: self.configuration,
1190 explicit: self.explicit,
1191 checkpoint: self.checkpoint,
1192 states: PhantomData,
1193 }
1194 }
1195
1196 pub fn prefetch_factor(mut self, factor: usize) -> Self {
1210 self.configuration.prefetch_factor = Some(factor);
1211 self.explicit.prefetch_factor = true;
1212 self
1213 }
1214
1215 pub fn prefetch_bytes(
1243 mut self,
1244 limit: NonZeroUsize,
1245 ) -> DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, N, K> {
1246 self.configuration.prefetch_bytes = Some(limit);
1247 self.explicit.prefetch_bytes = true;
1248 DataLoaderBuilder {
1249 dataset: self.dataset,
1250 plan: self.plan,
1251 collator: self.collator,
1252 transform_factory: self.transform_factory,
1253 worker_init: self.worker_init,
1254 configuration: self.configuration,
1255 explicit: self.explicit,
1256 checkpoint: self.checkpoint,
1257 states: PhantomData,
1258 }
1259 }
1260
1261 pub fn batch_sampler<B>(
1263 mut self,
1264 batches: B,
1265 ) -> DataLoaderBuilder<D, ExplicitBatches<B>, C, F, I, X, M, N, K> {
1266 self.explicit.batch_sampler = true;
1267 self.map(|_, collator| (ExplicitBatches { batches }, collator))
1268 }
1269
1270 #[allow(clippy::type_complexity)]
1278 fn build_inner(
1279 mut self,
1280 footprint: Option<FootprintFn<D, F>>,
1281 ) -> Result<OwnedDataLoader<D, P, C, F, I, X, M, N, K>>
1282 where
1283 D: Dataset,
1284 F: TransformFactory<D::Sample>,
1285 P: LoaderPlanConfiguration,
1286 I: WorkerInit,
1287 M: MemoryPolicy,
1288 {
1289 validate_configuration(&self.configuration, self.explicit)?;
1290 let pin_memory_status = resolve_pin_request(self.configuration.pin_memory)?;
1291 let batch_size = validate_batch_size(self.configuration.batch_size)?;
1292 let effective_prefetch = match (
1293 self.configuration.workers,
1294 self.configuration.prefetch_factor,
1295 ) {
1296 (0, _) => None,
1297 (_, Some(factor)) => NonZeroUsize::new(factor),
1298 (_, None) => NonZeroUsize::new(2),
1299 };
1300 let outstanding_capacity = match (self.configuration.workers, effective_prefetch) {
1301 (0, _) => None,
1302 (workers, Some(factor)) => {
1303 Some(workers.checked_mul(factor.get()).ok_or_else(|| {
1304 invalid_configuration(
1305 "prefetch_factor",
1306 "workers multiplied by prefetch_factor exceeds usize",
1307 )
1308 })?)
1309 }
1310 (_, None) => unreachable!("positive workers always have an effective prefetch factor"),
1311 };
1312 if let Some(capacity) = outstanding_capacity {
1313 validate_worker_pool_capacity::<D, F, I, M>(
1314 self.configuration.workers,
1315 effective_prefetch
1316 .expect("positive workers have a prefetch factor")
1317 .get(),
1318 capacity,
1319 )?;
1320 }
1321 self.plan
1322 .apply_batch_options(batch_size, self.configuration.drop_last);
1323 self.plan.apply_epoch(self.configuration.epoch);
1324 Ok(OwnedDataLoader {
1325 dataset: Arc::new(self.dataset),
1326 plan: self.plan,
1327 collator: self.collator,
1328 transform_factory: Arc::new(self.transform_factory),
1329 worker_init: Arc::new(self.worker_init),
1330 workers: self.configuration.workers,
1331 persistent_workers: self.configuration.persistent_workers,
1332 timeout: self.configuration.timeout,
1333 ordered: self.configuration.ordered,
1334 pin_memory_status,
1335 footprint,
1336 effective_prefetch_bytes: self.configuration.prefetch_bytes,
1337 effective_prefetch,
1338 outstanding_capacity,
1339 loader_seed: self.configuration.loader_seed,
1340 rank: self.configuration.rank,
1341 next_generation: 0,
1342 persistent_pool: None,
1343 checkpoint: self.checkpoint,
1344 policies: PhantomData,
1345 })
1346 }
1347}
1348
1349impl<D, P, C, F, I, X, M, N> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointDisabled> {
1350 pub fn dataset_identity(
1360 self,
1361 identity: String,
1362 ) -> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointFresh> {
1363 DataLoaderBuilder {
1364 dataset: self.dataset,
1365 plan: self.plan,
1366 collator: self.collator,
1367 transform_factory: self.transform_factory,
1368 worker_init: self.worker_init,
1369 configuration: self.configuration,
1370 explicit: self.explicit,
1371 checkpoint: CheckpointFresh { identity },
1372 states: PhantomData,
1373 }
1374 }
1375}
1376
1377impl<D, P, C, F, I, X, M, N> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointFresh> {
1378 pub fn resume_from<S>(
1386 self,
1387 state: S,
1388 ) -> DataLoaderBuilder<D, P, C, F, I, X, M, N, CheckpointResume<S>> {
1389 DataLoaderBuilder {
1390 dataset: self.dataset,
1391 plan: self.plan,
1392 collator: self.collator,
1393 transform_factory: self.transform_factory,
1394 worker_init: self.worker_init,
1395 configuration: self.configuration,
1396 explicit: self.explicit,
1397 checkpoint: CheckpointResume {
1398 identity: self.checkpoint.identity,
1399 state,
1400 },
1401 states: PhantomData,
1402 }
1403 }
1404}
1405
1406fn resident_bytes<T: MemoryFootprint>(values: &[T]) -> usize {
1407 let mut total = 0usize;
1408 for value in values {
1409 let Some(next) = total.checked_add(value.resident_bytes()) else {
1410 return usize::MAX;
1411 };
1412 total = next;
1413 }
1414 total
1415}
1416
1417impl<D, P, C, F, I, X>
1418 DataLoaderBuilder<D, P, C, F, I, X, MemoryDisabled, PinDisabled, CheckpointDisabled>
1419{
1420 #[allow(clippy::type_complexity)]
1422 pub fn build(
1423 self,
1424 ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryDisabled, PinDisabled, CheckpointDisabled>>
1425 where
1426 D: Dataset,
1427 F: TransformFactory<D::Sample>,
1428 P: LoaderPlanConfiguration,
1429 I: WorkerInit,
1430 {
1431 self.build_inner(None)
1432 }
1433}
1434
1435impl<D, P, C, F, I, X>
1436 DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, PinDisabled, CheckpointDisabled>
1437{
1438 #[allow(clippy::type_complexity)]
1440 pub fn build(
1441 self,
1442 ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryEnabled, PinDisabled, CheckpointDisabled>>
1443 where
1444 D: Dataset,
1445 F: TransformFactory<D::Sample>,
1446 TransformOutput<D, F>: MemoryFootprint,
1447 P: LoaderPlanConfiguration,
1448 I: WorkerInit,
1449 {
1450 self.build_inner(Some(resident_bytes::<TransformOutput<D, F>>))
1451 }
1452}
1453
1454impl<D, P, C, F, I, X, Q>
1455 DataLoaderBuilder<D, P, C, F, I, X, MemoryDisabled, PinEnabled<Q>, CheckpointDisabled>
1456{
1457 #[allow(clippy::type_complexity)]
1459 pub fn build(
1460 self,
1461 ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryDisabled, PinEnabled<Q>, CheckpointDisabled>>
1462 where
1463 D: Dataset,
1464 F: TransformFactory<D::Sample>,
1465 P: LoaderPlanConfiguration + LoaderPlan<TransformOutput<D, F>, C>,
1466 P::Batch: PinMemory,
1467 I: WorkerInit,
1468 {
1469 self.build_inner(None)
1470 }
1471}
1472
1473impl<D, P, C, F, I, X, Q>
1474 DataLoaderBuilder<D, P, C, F, I, X, MemoryEnabled, PinEnabled<Q>, CheckpointDisabled>
1475{
1476 #[allow(clippy::type_complexity)]
1478 pub fn build(
1479 self,
1480 ) -> Result<OwnedDataLoader<D, P, C, F, I, X, MemoryEnabled, PinEnabled<Q>, CheckpointDisabled>>
1481 where
1482 D: Dataset,
1483 F: TransformFactory<D::Sample>,
1484 TransformOutput<D, F>: MemoryFootprint,
1485 P: LoaderPlanConfiguration + LoaderPlan<TransformOutput<D, F>, C>,
1486 P::Batch: PinMemory,
1487 I: WorkerInit,
1488 {
1489 self.build_inner(Some(resident_bytes::<TransformOutput<D, F>>))
1490 }
1491}
1492
1493#[allow(private_bounds)]
1494impl<D, P, C, F, I, N>
1495 DataLoaderBuilder<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointFresh>
1496where
1497 D: DatasetCheckpoint,
1498 F: TransformFactory<D::Sample>,
1499 F::Transform: WorkerCheckpoint,
1500 P: LoaderPlanConfiguration
1501 + LoaderPlan<TransformOutput<D, F>, C>
1502 + CheckpointPlan<TransformOutput<D, F>, C>,
1503 I: WorkerInit,
1504 N: MapPinPolicy<D, P, C, F>,
1505{
1506 #[allow(clippy::type_complexity)]
1508 pub fn build(
1509 mut self,
1510 ) -> Result<
1511 OwnedDataLoader<
1512 D,
1513 P,
1514 C,
1515 F,
1516 I,
1517 SerialExecution,
1518 MemoryDisabled,
1519 N,
1520 CheckpointActive<P::Iter, F::Transform>,
1521 >,
1522 > {
1523 validate_exact_builder(
1524 &self.configuration,
1525 self.explicit,
1526 &self.checkpoint.identity,
1527 )?;
1528 let pin_memory_status = resolve_pin_request(self.configuration.pin_memory)?;
1529 let batch_size = validate_batch_size(self.configuration.batch_size)?;
1530 self.plan
1531 .apply_batch_options(batch_size, self.configuration.drop_last);
1532 self.plan.apply_epoch(self.configuration.epoch);
1533 let identity = self.plan.checkpoint_identity();
1534 validate_distributed_rank(&identity, self.configuration.rank)?;
1535 let checkpoint_configuration =
1536 checkpoint_configuration(&identity, &self.configuration, pin_memory_status)?;
1537
1538 Ok(OwnedDataLoader {
1539 dataset: Arc::new(self.dataset),
1540 plan: self.plan,
1541 collator: self.collator,
1542 transform_factory: Arc::new(self.transform_factory),
1543 worker_init: Arc::new(self.worker_init),
1544 workers: 0,
1545 persistent_workers: false,
1546 timeout: None,
1547 ordered: true,
1548 pin_memory_status,
1549 footprint: None,
1550 effective_prefetch_bytes: None,
1551 effective_prefetch: None,
1552 outstanding_capacity: None,
1553 loader_seed: self.configuration.loader_seed,
1554 rank: self.configuration.rank,
1555 next_generation: 0,
1556 persistent_pool: None,
1557 checkpoint: CheckpointActive {
1558 identity: self.checkpoint.identity,
1559 configuration: checkpoint_configuration,
1560 pending: None,
1561 },
1562 policies: PhantomData,
1563 })
1564 }
1565}
1566
1567#[allow(private_bounds)]
1568impl<D, P, C, F, I, N, DS, SS, TS, CS>
1569 DataLoaderBuilder<
1570 D,
1571 P,
1572 C,
1573 F,
1574 I,
1575 SerialExecution,
1576 MemoryDisabled,
1577 N,
1578 CheckpointResume<LoaderState<DS, SS, TS, CS>>,
1579 >
1580where
1581 D: DatasetCheckpoint<State = DS>,
1582 F: TransformFactory<D::Sample>,
1583 F::Transform: WorkerCheckpoint<State = TS>,
1584 P: LoaderPlanConfiguration
1585 + LoaderPlan<TransformOutput<D, F>, C>
1586 + CheckpointPlan<TransformOutput<D, F>, C, State = SS, CoordinatorState = CS>,
1587 I: WorkerInit,
1588 N: MapPinPolicy<D, P, C, F>,
1589{
1590 #[allow(clippy::type_complexity)]
1592 pub fn build(
1593 mut self,
1594 ) -> std::result::Result<
1595 OwnedDataLoader<
1596 D,
1597 P,
1598 C,
1599 F,
1600 I,
1601 SerialExecution,
1602 MemoryDisabled,
1603 N,
1604 CheckpointActive<P::Iter, F::Transform>,
1605 >,
1606 CheckpointBuildError<F::Error>,
1607 > {
1608 let CheckpointResume {
1609 identity: requested_identity,
1610 state,
1611 } = self.checkpoint;
1612 validate_exact_builder(&self.configuration, self.explicit, &requested_identity)?;
1613 let pin_memory_status = resolve_pin_request(self.configuration.pin_memory)?;
1614 let batch_size = validate_batch_size(self.configuration.batch_size)?;
1615 validate_static_loader_state_envelope(
1616 &state,
1617 &requested_identity,
1618 &self.configuration,
1619 pin_memory_status,
1620 <P as CheckpointPlan<TransformOutput<D, F>, C>>::AUTOMATIC_BATCHING,
1621 0,
1622 )?;
1623
1624 let plan_identity =
1625 <P as CheckpointPlan<TransformOutput<D, F>, C>>::checkpoint_identity_from_state(
1626 &state.sampler,
1627 batch_size,
1628 self.configuration.drop_last,
1629 )?;
1630 validate_distributed_rank(&plan_identity, self.configuration.rank)?;
1631 let expected_configuration =
1632 checkpoint_configuration(&plan_identity, &self.configuration, pin_memory_status)?;
1633 validate_plan_loader_state_envelope(&state, &expected_configuration)?;
1634
1635 self.dataset.validate_dataset_state(&state.dataset)?;
1636 self.plan.validate_checkpoint_state_with_batch_options(
1637 &state.sampler,
1638 state.epoch,
1639 state.next_batch,
1640 state.next_logical_sample,
1641 batch_size,
1642 self.configuration.drop_last,
1643 )?;
1644 let mut transform = self
1645 .transform_factory
1646 .create(None)
1647 .map_err(CheckpointBuildError::TransformFactory)?;
1648 transform.validate_snapshot(&state.transform)?;
1649 self.plan
1650 .validate_coordinator(&self.collator, &state.collate)?;
1651
1652 self.dataset.restore_dataset_validated(&state.dataset);
1653 self.plan
1654 .apply_batch_options(batch_size, self.configuration.drop_last);
1655 let batches = self.plan.restore_checkpoint_iter_validated(&state.sampler);
1656 transform.restore_validated(&state.transform);
1657 self.plan
1658 .restore_coordinator_validated(&mut self.collator, &state.collate);
1659
1660 Ok(OwnedDataLoader {
1661 dataset: Arc::new(self.dataset),
1662 plan: self.plan,
1663 collator: self.collator,
1664 transform_factory: Arc::new(self.transform_factory),
1665 worker_init: Arc::new(self.worker_init),
1666 workers: 0,
1667 persistent_workers: false,
1668 timeout: None,
1669 ordered: true,
1670 pin_memory_status,
1671 footprint: None,
1672 effective_prefetch_bytes: None,
1673 effective_prefetch: None,
1674 outstanding_capacity: None,
1675 loader_seed: self.configuration.loader_seed,
1676 rank: self.configuration.rank,
1677 next_generation: 0,
1678 persistent_pool: None,
1679 checkpoint: CheckpointActive {
1680 identity: requested_identity,
1681 configuration: expected_configuration,
1682 pending: Some(crate::checkpoint::PendingSerial {
1683 batches,
1684 transform,
1685 next_batch: state.next_batch,
1686 next_logical_sample: state.next_logical_sample,
1687 }),
1688 },
1689 policies: PhantomData,
1690 })
1691 }
1692}
1693
1694impl<D, S, C, F, I, X, M, N, K> DataLoaderBuilder<D, AutoBatch<S>, C, F, I, X, M, N, K>
1695where
1696 D: Dataset,
1697{
1698 pub fn sampler<S2>(
1700 mut self,
1701 sampler: S2,
1702 ) -> DataLoaderBuilder<D, AutoBatch<S2>, C, F, I, X, M, N, K> {
1703 self.explicit.sampler = true;
1704 self.map(|plan, collator| {
1705 (
1706 AutoBatch {
1707 sampler,
1708 batch_size: plan.batch_size,
1709 drop_last: plan.drop_last,
1710 },
1711 collator,
1712 )
1713 })
1714 }
1715
1716 #[allow(clippy::type_complexity)]
1723 pub fn shuffle(
1724 mut self,
1725 seed: u64,
1726 ) -> Result<DataLoaderBuilder<D, AutoBatch<RandomSampler>, C, F, I, X, M, N, K>> {
1727 self.explicit.shuffle = true;
1728 let sampler = RandomSampler::new(self.dataset.len(), seed)?;
1729 Ok(self.map(|plan, collator| {
1730 (
1731 AutoBatch {
1732 sampler,
1733 batch_size: plan.batch_size,
1734 drop_last: plan.drop_last,
1735 },
1736 collator,
1737 )
1738 }))
1739 }
1740
1741 pub fn without_batching(
1743 mut self,
1744 ) -> DataLoaderBuilder<D, NoBatch<S, DefaultConverter>, C, F, I, X, M, N, K> {
1745 self.explicit.without_batching = true;
1746 self.map(|plan, collator| {
1747 (
1748 NoBatch {
1749 sampler: plan.sampler,
1750 converter: DefaultConverter,
1751 },
1752 collator,
1753 )
1754 })
1755 }
1756
1757 pub fn collate<C2>(
1759 self,
1760 collator: C2,
1761 ) -> DataLoaderBuilder<D, AutoBatch<S>, C2, F, I, X, M, N, K> {
1762 self.map(|plan, _| (plan, collator))
1763 }
1764}
1765
1766impl<D, B, C, F, I, X, M, N, K> DataLoaderBuilder<D, ExplicitBatches<B>, C, F, I, X, M, N, K>
1767where
1768 D: Dataset,
1769{
1770 pub fn sampler<S>(
1772 mut self,
1773 sampler: S,
1774 ) -> DataLoaderBuilder<D, AutoBatch<S>, C, F, I, X, M, N, K> {
1775 self.explicit.sampler = true;
1776 let batch_size =
1777 NonZeroUsize::new(self.configuration.batch_size).unwrap_or(NonZeroUsize::MIN);
1778 let drop_last = self.configuration.drop_last;
1779 self.map(|_, collator| {
1780 (
1781 AutoBatch {
1782 sampler,
1783 batch_size,
1784 drop_last,
1785 },
1786 collator,
1787 )
1788 })
1789 }
1790
1791 #[allow(clippy::type_complexity)]
1798 pub fn shuffle(
1799 mut self,
1800 seed: u64,
1801 ) -> Result<DataLoaderBuilder<D, AutoBatch<RandomSampler>, C, F, I, X, M, N, K>> {
1802 self.explicit.shuffle = true;
1803 let sampler = RandomSampler::new(self.dataset.len(), seed)?;
1804 let batch_size =
1805 NonZeroUsize::new(self.configuration.batch_size).unwrap_or(NonZeroUsize::MIN);
1806 let drop_last = self.configuration.drop_last;
1807 Ok(self.map(|_, collator| {
1808 (
1809 AutoBatch {
1810 sampler,
1811 batch_size,
1812 drop_last,
1813 },
1814 collator,
1815 )
1816 }))
1817 }
1818
1819 pub fn without_batching(
1821 mut self,
1822 ) -> DataLoaderBuilder<D, NoBatch<SequentialSampler, DefaultConverter>, C, F, I, X, M, N, K>
1823 {
1824 self.explicit.without_batching = true;
1825 let length = self.dataset.len();
1826 self.map(|_, collator| {
1827 (
1828 NoBatch {
1829 sampler: SequentialSampler::new(length),
1830 converter: DefaultConverter,
1831 },
1832 collator,
1833 )
1834 })
1835 }
1836
1837 pub fn collate<C2>(
1839 self,
1840 collator: C2,
1841 ) -> DataLoaderBuilder<D, ExplicitBatches<B>, C2, F, I, X, M, N, K> {
1842 self.map(|plan, _| (plan, collator))
1843 }
1844}
1845
1846impl<D, S, V, C, F, I, X, M, N, K> DataLoaderBuilder<D, NoBatch<S, V>, C, F, I, X, M, N, K>
1847where
1848 D: Dataset,
1849{
1850 pub fn sampler<S2>(
1852 mut self,
1853 sampler: S2,
1854 ) -> DataLoaderBuilder<D, NoBatch<S2, V>, C, F, I, X, M, N, K> {
1855 self.explicit.sampler = true;
1856 self.map(|plan, collator| {
1857 (
1858 NoBatch {
1859 sampler,
1860 converter: plan.converter,
1861 },
1862 collator,
1863 )
1864 })
1865 }
1866
1867 #[allow(clippy::type_complexity)]
1874 pub fn shuffle(
1875 mut self,
1876 seed: u64,
1877 ) -> Result<DataLoaderBuilder<D, NoBatch<RandomSampler, V>, C, F, I, X, M, N, K>> {
1878 self.explicit.shuffle = true;
1879 let sampler = RandomSampler::new(self.dataset.len(), seed)?;
1880 Ok(self.map(|plan, collator| {
1881 (
1882 NoBatch {
1883 sampler,
1884 converter: plan.converter,
1885 },
1886 collator,
1887 )
1888 }))
1889 }
1890
1891 pub fn convert<V2>(
1893 self,
1894 converter: V2,
1895 ) -> DataLoaderBuilder<D, NoBatch<S, V2>, C, F, I, X, M, N, K> {
1896 self.map(|plan, collator| {
1897 (
1898 NoBatch {
1899 sampler: plan.sampler,
1900 converter,
1901 },
1902 collator,
1903 )
1904 })
1905 }
1906}
1907
1908#[allow(private_bounds)]
1925pub struct OwnedDataLoader<
1926 D,
1927 P,
1928 C,
1929 F = IdentityTransformFactory,
1930 I = NoWorkerInit,
1931 X = SerialExecution,
1932 M = MemoryDisabled,
1933 N = PinDisabled,
1934 K = CheckpointDisabled,
1935> where
1936 D: Dataset,
1937 F: TransformFactory<D::Sample>,
1938 I: WorkerInit,
1939 M: MemoryPolicy,
1940{
1941 dataset: Arc<D>,
1942 plan: P,
1943 collator: C,
1944 transform_factory: Arc<F>,
1945 worker_init: Arc<I>,
1946 workers: usize,
1947 persistent_workers: bool,
1948 timeout: Option<Duration>,
1949 ordered: bool,
1950 pin_memory_status: PinMemoryStatus,
1951 footprint: Option<FootprintFn<D, F>>,
1952 effective_prefetch_bytes: Option<NonZeroUsize>,
1953 effective_prefetch: Option<NonZeroUsize>,
1954 outstanding_capacity: Option<usize>,
1955 loader_seed: u64,
1956 rank: usize,
1957 next_generation: u64,
1958 persistent_pool: Option<WorkerPool<D, F, I, M>>,
1959 checkpoint: K,
1960 policies: PhantomData<(X, M, N)>,
1961}
1962
1963#[allow(private_bounds)]
1964impl<D, P, C, F, I, M, N> OwnedDataLoader<D, P, C, F, I, SerialExecution, M, N, CheckpointDisabled>
1965where
1966 D: Dataset,
1967 F: TransformFactory<D::Sample>,
1968 P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
1969 I: WorkerInit,
1970 M: MemoryPolicy,
1971 N: MapPinPolicy<D, P, C, F>,
1972{
1973 pub fn iter(&mut self) -> LoaderIter<'_, D, P, C, F, I, M, N, CheckpointDisabled> {
1975 let (transform, transform_error) = match self.transform_factory.create(None) {
1976 Ok(transform) => (Some(transform), None),
1977 Err(error) => (None, Some(error)),
1978 };
1979 let batches = transform.as_ref().map(|_| self.plan.iter());
1980 let epoch = self.plan.epoch();
1981 let run_context = WorkerRunContext::new(0, self.loader_seed, epoch);
1982 LoaderIter {
1983 dataset: self.dataset.as_ref(),
1984 plan: &mut self.plan,
1985 collator: &mut self.collator,
1986 batches,
1987 transform,
1988 transform_error,
1989 next_batch: 0,
1990 next_logical_sample: 0,
1991 exhausted: false,
1992 loader_seed: self.loader_seed,
1993 epoch,
1994 rank: self.rank,
1995 run_context,
1996 pin_memory_status: self.pin_memory_status,
1997 checkpoint: CheckpointDisabled,
1998 output: PhantomData,
1999 policies: PhantomData,
2000 }
2001 }
2002}
2003
2004#[allow(private_bounds, private_interfaces)]
2005impl<D, P, C, F, I, N>
2006 OwnedDataLoader<
2007 D,
2008 P,
2009 C,
2010 F,
2011 I,
2012 SerialExecution,
2013 MemoryDisabled,
2014 N,
2015 CheckpointActive<P::Iter, F::Transform>,
2016 >
2017where
2018 D: DatasetCheckpoint,
2019 F: TransformFactory<D::Sample>,
2020 F::Transform: WorkerCheckpoint,
2021 P: LoaderPlan<TransformOutput<D, F>, C> + CheckpointPlan<TransformOutput<D, F>, C>,
2022 I: WorkerInit,
2023 N: MapPinPolicy<D, P, C, F>,
2024{
2025 pub fn iter(
2027 &mut self,
2028 ) -> LoaderIter<'_, D, P, C, F, I, MemoryDisabled, N, CheckpointIteration> {
2029 let pending = self.checkpoint.pending.take();
2030 let (batches, transform, transform_error, next_batch, next_logical_sample) = match pending {
2031 Some(pending) => (
2032 Some(pending.batches),
2033 Some(pending.transform),
2034 None,
2035 pending.next_batch,
2036 pending.next_logical_sample,
2037 ),
2038 None => match self.transform_factory.create(None) {
2039 Ok(transform) => (Some(self.plan.iter()), Some(transform), None, 0, 0),
2040 Err(error) => (None, None, Some(error), 0, 0),
2041 },
2042 };
2043 let epoch = self.plan.epoch();
2044 let run_context = WorkerRunContext::new(0, self.loader_seed, epoch);
2045 LoaderIter {
2046 dataset: self.dataset.as_ref(),
2047 plan: &mut self.plan,
2048 collator: &mut self.collator,
2049 batches,
2050 transform,
2051 transform_error,
2052 next_batch,
2053 next_logical_sample,
2054 exhausted: false,
2055 loader_seed: self.loader_seed,
2056 epoch,
2057 rank: self.rank,
2058 run_context,
2059 pin_memory_status: self.pin_memory_status,
2060 checkpoint: CheckpointIteration {
2061 identity: self.checkpoint.identity.clone(),
2062 configuration: self.checkpoint.configuration.clone(),
2063 boundary_valid: true,
2064 },
2065 output: PhantomData,
2066 policies: PhantomData,
2067 }
2068 }
2069
2070 pub fn set_epoch(&mut self, epoch: u64) {
2076 self.plan.set_epoch(epoch);
2077 self.checkpoint.pending = None;
2078 }
2079}
2080
2081#[allow(private_bounds)]
2082impl<D, P, C, F, I, X, M, N> OwnedDataLoader<D, P, C, F, I, X, M, N, CheckpointDisabled>
2083where
2084 D: Dataset,
2085 F: TransformFactory<D::Sample>,
2086 P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2087 I: WorkerInit,
2088 M: MemoryPolicy,
2089{
2090 pub fn set_epoch(&mut self, epoch: u64) {
2092 self.plan.set_epoch(epoch);
2093 }
2094}
2095
2096#[allow(private_bounds)]
2097impl<D, P, C, F, I, X, M, N, K> OwnedDataLoader<D, P, C, F, I, X, M, N, K>
2098where
2099 D: Dataset,
2100 F: TransformFactory<D::Sample>,
2101 P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2102 I: WorkerInit,
2103 M: MemoryPolicy,
2104{
2105 pub fn len(&self) -> Option<usize> {
2107 self.plan.exact_len()
2108 }
2109
2110 pub fn is_empty(&self) -> Option<bool> {
2112 self.len().map(|length| length == 0)
2113 }
2114
2115 pub fn epoch(&self) -> u64 {
2117 self.plan.epoch()
2118 }
2119
2120 pub fn workers(&self) -> usize {
2122 self.workers
2123 }
2124
2125 pub fn effective_prefetch_factor(&self) -> Option<NonZeroUsize> {
2127 self.effective_prefetch
2128 }
2129
2130 pub fn is_ordered(&self) -> bool {
2132 self.ordered
2133 }
2134
2135 pub fn pin_memory_enabled(&self) -> bool {
2137 !matches!(self.pin_memory_status, PinMemoryStatus::Disabled)
2138 }
2139
2140 pub fn pin_memory_status(&self) -> PinMemoryStatus {
2142 self.pin_memory_status
2143 }
2144
2145 pub fn effective_prefetch_bytes(&self) -> Option<NonZeroUsize> {
2147 self.effective_prefetch_bytes
2148 }
2149
2150 pub fn timeout(&self) -> Option<Duration> {
2152 self.timeout
2153 }
2154
2155 pub fn persistent_workers_enabled(&self) -> bool {
2157 self.persistent_workers
2158 }
2159}
2160
2161pub struct LoaderIter<
2169 'a,
2170 D,
2171 P,
2172 C,
2173 F = IdentityTransformFactory,
2174 I = NoWorkerInit,
2175 M = MemoryDisabled,
2176 N = PinDisabled,
2177 K = CheckpointDisabled,
2178> where
2179 D: Dataset,
2180 F: TransformFactory<D::Sample>,
2181 P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2182 I: WorkerInit,
2183{
2184 dataset: &'a D,
2185 plan: &'a mut P,
2186 collator: &'a mut C,
2187 batches: Option<P::Iter>,
2188 transform: Option<F::Transform>,
2189 transform_error: Option<F::Error>,
2190 next_batch: u64,
2191 next_logical_sample: u64,
2192 exhausted: bool,
2193 loader_seed: u64,
2194 epoch: u64,
2195 rank: usize,
2196 run_context: WorkerRunContext,
2197 pin_memory_status: PinMemoryStatus,
2198 checkpoint: K,
2199 output: PhantomData<fn() -> I>,
2200 policies: PhantomData<(M, N)>,
2201}
2202
2203#[allow(private_bounds, private_interfaces)]
2204impl<D, P, C, F, I, N> LoaderIter<'_, D, P, C, F, I, MemoryDisabled, N, CheckpointIteration>
2205where
2206 D: DatasetCheckpoint,
2207 F: TransformFactory<D::Sample>,
2208 F::Transform: WorkerCheckpoint,
2209 P: LoaderPlan<TransformOutput<D, F>, C> + CheckpointPlan<TransformOutput<D, F>, C>,
2210 I: WorkerInit,
2211 N: MapPinPolicy<D, P, C, F>,
2212{
2213 #[allow(clippy::type_complexity)]
2220 pub fn checkpoint(
2221 &mut self,
2222 ) -> std::result::Result<
2223 LoaderState<
2224 D::State,
2225 P::State,
2226 <F::Transform as WorkerCheckpoint>::State,
2227 P::CoordinatorState,
2228 >,
2229 IterError<D, P, C, F, I>,
2230 > {
2231 if !self.checkpoint.boundary_valid {
2232 return Err(LoaderError::Checkpoint {
2233 reason: "the active iterator is not at a consumer-visible batch boundary"
2234 .to_owned(),
2235 });
2236 }
2237 let transform = self
2238 .transform
2239 .as_ref()
2240 .ok_or_else(|| LoaderError::Checkpoint {
2241 reason: "the serial transform was not constructed".to_owned(),
2242 })?;
2243 let sampler = self
2244 .plan
2245 .checkpoint_state(self.next_batch, self.next_logical_sample)
2246 .map_err(LoaderError::Configuration)?;
2247 Ok(LoaderState {
2248 schema_version: LOADER_STATE_SCHEMA_VERSION,
2249 dataset_identity: self.checkpoint.identity.clone(),
2250 epoch: self.epoch,
2251 iterator_generation: 0,
2252 next_batch: self.next_batch,
2253 next_logical_sample: self.next_logical_sample,
2254 dataset: self.dataset.snapshot_dataset(),
2255 sampler,
2256 transform: transform.snapshot(),
2257 collate: self.plan.save_coordinator(self.collator),
2258 rng_derivation_version: TASK_RNG_DERIVATION_VERSION,
2259 worker_seed_derivation_version: WORKER_SEED_DERIVATION_VERSION,
2260 configuration: self.checkpoint.configuration.clone(),
2261 })
2262 }
2263}
2264
2265impl<D, P, C, F, I, M, N, K> Iterator for LoaderIter<'_, D, P, C, F, I, M, N, K>
2266where
2267 D: Dataset,
2268 F: TransformFactory<D::Sample>,
2269 P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2270 I: WorkerInit,
2271 N: MapPinPolicy<D, P, C, F>,
2272 K: SerialBoundaryPolicy,
2273{
2274 type Item = std::result::Result<
2275 P::Batch,
2276 LoaderError<
2277 PipelineError<
2278 D::Error,
2279 <F::Transform as Transform<D::Sample>>::Error,
2280 P::Error,
2281 F::Error,
2282 I::Error,
2283 >,
2284 >,
2285 >;
2286
2287 fn next(&mut self) -> Option<Self::Item> {
2288 if self.exhausted {
2289 return None;
2290 }
2291 self.checkpoint.begin_next();
2292 if let Some(source) = self.transform_error.take() {
2293 self.exhausted = true;
2294 return Some(Err(LoaderError::Pipeline {
2295 batch: None,
2296 worker: None,
2297 source: PipelineError::TransformInit(source),
2298 }));
2299 }
2300
2301 let indices = match self.batches.as_mut().and_then(Iterator::next) {
2302 Some(indices) => indices,
2303 None => {
2304 self.exhausted = true;
2305 return None;
2306 }
2307 };
2308 let batch = self.next_batch;
2309 let samples = match self.dataset.get_batch(&indices) {
2310 Ok(samples) => samples,
2311 Err(source) => {
2312 self.exhausted = true;
2313 return Some(Err(LoaderError::Pipeline {
2314 batch: Some(batch),
2315 worker: None,
2316 source: PipelineError::Dataset(source),
2317 }));
2318 }
2319 };
2320 if samples.len() != indices.len() {
2321 self.exhausted = true;
2322 return Some(Err(LoaderError::InvalidBatchCardinality {
2323 batch,
2324 worker: None,
2325 expected: indices.len(),
2326 actual: samples.len(),
2327 }));
2328 }
2329 let mut transformed = Vec::with_capacity(samples.len());
2330 for sample in samples {
2331 let context = TaskContext {
2332 loader_seed: self.loader_seed,
2333 epoch: self.epoch,
2334 rank: self.rank,
2335 logical_sample: self.next_logical_sample,
2336 stage: 0,
2337 cancellation: self.run_context.cancellation.clone(),
2338 deadline: self.run_context.deadline.clone(),
2339 };
2340 let Some(transform) = self.transform.as_mut() else {
2341 self.exhausted = true;
2342 return None;
2343 };
2344 match transform.transform(sample, &context) {
2345 Ok(sample) => transformed.push(sample),
2346 Err(source) => {
2347 self.exhausted = true;
2348 return Some(Err(LoaderError::Pipeline {
2349 batch: Some(batch),
2350 worker: None,
2351 source: PipelineError::Transform(source),
2352 }));
2353 }
2354 }
2355 match self.next_logical_sample.checked_add(1) {
2356 Some(next) => self.next_logical_sample = next,
2357 None => self.exhausted = true,
2358 }
2359 }
2360 let result =
2361 self.plan
2362 .finish(self.collator, transformed)
2363 .map_err(|source| LoaderError::Pipeline {
2364 batch: Some(batch),
2365 worker: None,
2366 source: PipelineError::Collate(source),
2367 })
2368 .and_then(|value| match self.pin_memory_status {
2369 PinMemoryStatus::Enabled(device) => N::pin(value, device)
2370 .map_err(|source| LoaderError::PinMemory { batch, source }),
2371 PinMemoryStatus::Disabled | PinMemoryStatus::DisabledNoAccelerator => Ok(value),
2372 });
2373 if result.is_err() {
2374 self.exhausted = true;
2375 } else if let Some(next) = self.next_batch.checked_add(1) {
2376 self.next_batch = next;
2377 self.checkpoint.commit_visible();
2378 } else {
2379 self.exhausted = true;
2380 }
2381 Some(result)
2382 }
2383}
2384
2385#[allow(private_bounds)]
2393pub struct WorkerLoaderIter<
2394 'a,
2395 D,
2396 P,
2397 C,
2398 F = IdentityTransformFactory,
2399 I = NoWorkerInit,
2400 M = MemoryDisabled,
2401 N = PinDisabled,
2402 K = CheckpointDisabled,
2403> where
2404 D: Dataset,
2405 F: TransformFactory<D::Sample>,
2406 P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2407 I: WorkerInit,
2408 M: MemoryPolicy,
2409{
2410 dataset: Arc<D>,
2411 plan: &'a mut P,
2412 collator: &'a mut C,
2413 batches: Option<P::Iter>,
2414 serial_transform: Option<F::Transform>,
2415 serial_transform_error: Option<F::Error>,
2416 pool: Option<IteratorPool<'a, D, F, I, M>>,
2417 completed: BTreeMap<u64, CompletedBatch<D, F, M>>,
2418 pending_error: Option<IterError<D, P, C, F, I>>,
2419 generation: u64,
2420 run_context: Option<WorkerRunContext>,
2421 workers: usize,
2422 capacity: usize,
2423 ordered: bool,
2424 timeout: Option<Duration>,
2425 loader_seed: u64,
2426 epoch: u64,
2427 rank: usize,
2428 next_submission: u64,
2429 next_visible: u64,
2430 next_logical_sample: u64,
2431 pending_submission: Option<PendingSubmission>,
2432 outstanding: usize,
2433 source_exhausted: bool,
2434 submission_closed: bool,
2435 exhausted: bool,
2436 pin_memory_status: PinMemoryStatus,
2437 _checkpoint: K,
2438 policies: PhantomData<(M, N)>,
2439}
2440
2441enum IteratorPool<'a, D, F, I, M>
2442where
2443 D: Dataset,
2444 F: TransformFactory<D::Sample>,
2445 I: WorkerInit,
2446 M: MemoryPolicy,
2447{
2448 Owned(WorkerPool<D, F, I, M>),
2449 Persistent {
2450 pool: &'a mut WorkerPool<D, F, I, M>,
2451 generation: u64,
2452 },
2453}
2454
2455impl<D, F, I, M> IteratorPool<'_, D, F, I, M>
2456where
2457 D: Dataset,
2458 F: TransformFactory<D::Sample>,
2459 I: WorkerInit,
2460 M: MemoryPolicy,
2461{
2462 fn pool(&self) -> &WorkerPool<D, F, I, M> {
2463 match self {
2464 Self::Owned(pool) => pool,
2465 Self::Persistent { pool, .. } => pool,
2466 }
2467 }
2468
2469 fn pool_mut(&mut self) -> &mut WorkerPool<D, F, I, M> {
2470 match self {
2471 Self::Owned(pool) => pool,
2472 Self::Persistent { pool, .. } => pool,
2473 }
2474 }
2475
2476 fn is_persistent(&self) -> bool {
2477 matches!(self, Self::Persistent { .. })
2478 }
2479}
2480
2481impl<D, F, I, M> Drop for IteratorPool<'_, D, F, I, M>
2482where
2483 D: Dataset,
2484 F: TransformFactory<D::Sample>,
2485 I: WorkerInit,
2486 M: MemoryPolicy,
2487{
2488 fn drop(&mut self) {
2489 if let Self::Persistent { pool, generation } = self
2490 && pool.quiesce(*generation).is_err()
2491 {
2492 pool.poison();
2493 }
2494 }
2495}
2496
2497#[allow(private_bounds)]
2498impl<D, P, C, F, I, M, N> OwnedDataLoader<D, P, C, F, I, WorkerExecution, M, N, CheckpointDisabled>
2499where
2500 D: Dataset + Send + Sync + 'static,
2501 D::Sample: Send + 'static,
2502 D::Error: Send + 'static,
2503 F: TransformFactory<D::Sample> + Send + Sync + 'static,
2504 F::Transform: Send + 'static,
2505 <F::Transform as Transform<D::Sample>>::Output: Send + 'static,
2506 <F::Transform as Transform<D::Sample>>::Error: Send + 'static,
2507 F::Error: Send + 'static,
2508 P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2509 I: WorkerInit + Send + Sync + 'static,
2510 I::Error: Send + 'static,
2511 M: MemoryPolicy + Send + 'static,
2512 N: MapPinPolicy<D, P, C, F>,
2513{
2514 pub fn iter(&mut self) -> WorkerLoaderIter<'_, D, P, C, F, I, M, N, CheckpointDisabled> {
2517 let (epoch, epoch_panicked) = if self.workers == 0 {
2518 (self.plan.epoch(), false)
2519 } else {
2520 match catch_unwind(AssertUnwindSafe(|| self.plan.epoch())) {
2521 Ok(epoch) => (epoch, false),
2522 Err(_) => (0, true),
2523 }
2524 };
2525 let mut iterator = WorkerLoaderIter {
2526 dataset: Arc::clone(&self.dataset),
2527 plan: &mut self.plan,
2528 collator: &mut self.collator,
2529 batches: None,
2530 serial_transform: None,
2531 serial_transform_error: None,
2532 pool: None,
2533 completed: BTreeMap::new(),
2534 pending_error: None,
2535 generation: self.next_generation,
2536 run_context: Some(WorkerRunContext::new(
2537 self.next_generation,
2538 self.loader_seed,
2539 epoch,
2540 )),
2541 workers: self.workers,
2542 capacity: self.outstanding_capacity.unwrap_or(0),
2543 ordered: self.ordered,
2544 timeout: self.timeout,
2545 loader_seed: self.loader_seed,
2546 epoch,
2547 rank: self.rank,
2548 next_submission: 0,
2549 next_visible: 0,
2550 next_logical_sample: 0,
2551 pending_submission: None,
2552 outstanding: 0,
2553 source_exhausted: false,
2554 submission_closed: false,
2555 exhausted: false,
2556 pin_memory_status: self.pin_memory_status,
2557 _checkpoint: CheckpointDisabled,
2558 policies: PhantomData,
2559 };
2560
2561 if epoch_panicked {
2562 iterator.pending_error = Some(LoaderError::CoordinatorPanic {
2563 stage: P::EPOCH_PANIC_STAGE,
2564 batch: None,
2565 });
2566 return iterator;
2567 }
2568 if self.workers == 0 {
2569 match self.transform_factory.create(None) {
2570 Ok(transform) => {
2571 iterator.serial_transform = Some(transform);
2572 iterator.batches = Some(iterator.plan.iter());
2573 }
2574 Err(error) => iterator.serial_transform_error = Some(error),
2575 }
2576 return iterator;
2577 }
2578 let Some(next_generation) = self.next_generation.checked_add(1) else {
2579 iterator.pending_error = Some(LoaderError::Configuration(invalid_configuration(
2580 "workers",
2581 "iterator generation overflowed",
2582 )));
2583 return iterator;
2584 };
2585 iterator.batches = match catch_unwind(AssertUnwindSafe(|| iterator.plan.iter())) {
2586 Ok(batches) => Some(batches),
2587 Err(_) => {
2588 iterator.pending_error = Some(LoaderError::CoordinatorPanic {
2589 stage: P::CREATION_PANIC_STAGE,
2590 batch: None,
2591 });
2592 return iterator;
2593 }
2594 };
2595 self.next_generation = next_generation;
2596 let configuration = WorkerPoolConfiguration {
2597 workers: self.workers,
2598 prefetch_factor: self
2599 .effective_prefetch
2600 .expect("positive workers have a validated prefetch factor")
2601 .get(),
2602 result_capacity: iterator.capacity,
2603 seed_generation: iterator.generation,
2604 loader_seed: self.loader_seed,
2605 rank: self.rank,
2606 };
2607 let run_context =
2608 WorkerRunContext::new(iterator.generation, self.loader_seed, iterator.epoch)
2609 .with_byte_budget(
2610 self.effective_prefetch_bytes
2611 .map(|limit| ByteBudget::new(limit.get())),
2612 );
2613 iterator.run_context = Some(run_context.clone());
2614
2615 if self.persistent_workers {
2616 if self.persistent_pool.is_none() {
2617 match WorkerPool::new(
2618 Arc::clone(&self.dataset),
2619 Arc::clone(&self.transform_factory),
2620 Arc::clone(&self.worker_init),
2621 configuration,
2622 self.footprint,
2623 self.ordered,
2624 ) {
2625 Ok(pool) => self.persistent_pool = Some(pool),
2626 Err(error) => {
2627 iterator.pending_error = Some(LoaderError::Configuration(error));
2628 return iterator;
2629 }
2630 }
2631 }
2632 let pool = self
2633 .persistent_pool
2634 .as_mut()
2635 .expect("persistent pool was created");
2636 if pool.is_poisoned() || pool.start_generation(run_context).is_err() {
2637 iterator.pending_error = Some(LoaderError::ChannelClosed { batch: 0 });
2638 return iterator;
2639 }
2640 iterator.pool = Some(IteratorPool::Persistent {
2641 pool,
2642 generation: iterator.generation,
2643 });
2644 } else {
2645 match WorkerPool::new(
2646 Arc::clone(&self.dataset),
2647 Arc::clone(&self.transform_factory),
2648 Arc::clone(&self.worker_init),
2649 configuration,
2650 self.footprint,
2651 self.ordered,
2652 ) {
2653 Ok(mut pool) => {
2654 if pool.start_generation(run_context).is_err() {
2655 iterator.pending_error = Some(LoaderError::ChannelClosed { batch: 0 });
2656 } else {
2657 iterator.pool = Some(IteratorPool::Owned(pool));
2658 }
2659 }
2660 Err(error) => iterator.pending_error = Some(LoaderError::Configuration(error)),
2661 }
2662 }
2663 if iterator.pending_error.is_none() {
2664 iterator.fill_available();
2665 }
2666 iterator
2667 }
2668}
2669
2670#[allow(private_bounds)]
2671impl<D, P, C, F, I, M, N, K> WorkerLoaderIter<'_, D, P, C, F, I, M, N, K>
2672where
2673 D: Dataset + Send + Sync + 'static,
2674 D::Sample: Send + 'static,
2675 D::Error: Send + 'static,
2676 F: TransformFactory<D::Sample> + Send + Sync + 'static,
2677 F::Transform: Send + 'static,
2678 <F::Transform as Transform<D::Sample>>::Output: Send + 'static,
2679 <F::Transform as Transform<D::Sample>>::Error: Send + 'static,
2680 F::Error: Send + 'static,
2681 P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
2682 I: WorkerInit + Send + Sync + 'static,
2683 I::Error: Send + 'static,
2684 M: MemoryPolicy + Send + 'static,
2685 N: MapPinPolicy<D, P, C, F>,
2686 K: WorkerBoundaryPolicy<IterError<D, P, C, F, I>>,
2687{
2688 fn fill_available(&mut self) {
2689 while self.outstanding < self.capacity && !self.source_exhausted && !self.submission_closed
2690 {
2691 if !self.submit_one() {
2692 break;
2693 }
2694 }
2695 }
2696
2697 fn submit_one(&mut self) -> bool {
2698 if let Some(pending) = self.pending_submission.take() {
2699 return self.try_submit(pending);
2700 }
2701 let next_batch = catch_unwind(AssertUnwindSafe(|| {
2702 self.batches.as_mut().and_then(Iterator::next)
2703 }));
2704 let Some(indices) = (match next_batch {
2705 Ok(indices) => indices,
2706 Err(_) => {
2707 self.pending_error = Some(LoaderError::CoordinatorPanic {
2708 stage: P::REFILL_PANIC_STAGE,
2709 batch: Some(self.next_submission),
2710 });
2711 self.submission_closed = true;
2712 return false;
2713 }
2714 }) else {
2715 self.source_exhausted = true;
2716 return false;
2717 };
2718 let batch_sequence = self.next_submission;
2719 let Some(next_submission) = batch_sequence.checked_add(1) else {
2720 self.pending_error = Some(LoaderError::Configuration(invalid_configuration(
2721 "sampler",
2722 "batch sequence overflowed",
2723 )));
2724 self.submission_closed = true;
2725 return false;
2726 };
2727 let logical_start = self.next_logical_sample;
2728 let Some(logical_end) = logical_start.checked_add(indices.len() as u64) else {
2729 self.pending_error = Some(LoaderError::Configuration(invalid_configuration(
2730 "sampler",
2731 "logical sample identifier overflowed",
2732 )));
2733 self.submission_closed = true;
2734 return false;
2735 };
2736 let logical_samples = (logical_start..logical_end).collect();
2737 let worker = batch_sequence as usize % self.workers;
2738 self.try_submit(PendingSubmission {
2739 worker,
2740 task: WorkerTask {
2741 generation: self.generation,
2742 batch_sequence,
2743 logical_samples,
2744 indices,
2745 },
2746 next_submission,
2747 logical_end,
2748 })
2749 }
2750
2751 fn try_submit(&mut self, pending: PendingSubmission) -> bool {
2752 let PendingSubmission {
2753 worker,
2754 task,
2755 next_submission,
2756 logical_end,
2757 } = pending;
2758 match self
2759 .pool
2760 .as_ref()
2761 .expect("submissions require an active worker pool")
2762 .pool()
2763 .submit(worker, task)
2764 {
2765 WorkerSubmit::Submitted => {
2766 self._checkpoint.submitted(next_submission - 1, logical_end);
2767 self.next_submission = next_submission;
2768 self.next_logical_sample = logical_end;
2769 self.outstanding += 1;
2770 true
2771 }
2772 WorkerSubmit::Full(task) => {
2773 self.pending_submission = Some(PendingSubmission {
2774 worker,
2775 task,
2776 next_submission,
2777 logical_end,
2778 });
2779 false
2780 }
2781 WorkerSubmit::Closed => {
2782 self.submission_closed = true;
2783 false
2784 }
2785 }
2786 }
2787
2788 fn stop(&mut self) {
2789 self.stop_inner(false);
2790 }
2791
2792 fn stop_poisoned(&mut self) {
2793 self.stop_inner(true);
2794 }
2795
2796 fn stop_inner(&mut self, poisoned: bool) {
2797 self.exhausted = true;
2798 if let Some(run_context) = &self.run_context {
2799 run_context.cancel();
2800 }
2801 if let Some(mut pool) = self.pool.take() {
2802 let should_poison = poisoned
2803 || (pool.is_persistent() && pool.pool_mut().quiesce(self.generation).is_err());
2804 if should_poison {
2805 pool.pool_mut().poison();
2806 }
2807 }
2808 self.completed.clear();
2809 self.pending_submission = None;
2810 }
2811
2812 fn next_serial(&mut self) -> Option<IterResult<D, P, C, F, I>> {
2813 if let Some(source) = self.serial_transform_error.take() {
2814 self.exhausted = true;
2815 return Some(Err(LoaderError::Pipeline {
2816 batch: None,
2817 worker: None,
2818 source: PipelineError::TransformInit(source),
2819 }));
2820 }
2821 let indices = match self.batches.as_mut().and_then(Iterator::next) {
2822 Some(indices) => indices,
2823 None => {
2824 self.exhausted = true;
2825 return None;
2826 }
2827 };
2828 let batch = self.next_visible;
2829 let samples = match self.dataset.get_batch(&indices) {
2830 Ok(samples) => samples,
2831 Err(source) => {
2832 self.exhausted = true;
2833 return Some(Err(LoaderError::Pipeline {
2834 batch: Some(batch),
2835 worker: None,
2836 source: PipelineError::Dataset(source),
2837 }));
2838 }
2839 };
2840 if samples.len() != indices.len() {
2841 self.exhausted = true;
2842 return Some(Err(LoaderError::InvalidBatchCardinality {
2843 batch,
2844 worker: None,
2845 expected: indices.len(),
2846 actual: samples.len(),
2847 }));
2848 }
2849 let mut transformed = Vec::with_capacity(samples.len());
2850 for sample in samples {
2851 let context = TaskContext {
2852 loader_seed: self.loader_seed,
2853 epoch: self.epoch,
2854 rank: self.rank,
2855 logical_sample: self.next_logical_sample,
2856 stage: 0,
2857 cancellation: self
2858 .run_context
2859 .as_ref()
2860 .expect("serial generation context")
2861 .cancellation
2862 .clone(),
2863 deadline: self
2864 .run_context
2865 .as_ref()
2866 .expect("serial generation context")
2867 .deadline
2868 .clone(),
2869 };
2870 let transform = self
2871 .serial_transform
2872 .as_mut()
2873 .expect("serial transform exists after successful construction");
2874 match transform.transform(sample, &context) {
2875 Ok(sample) => transformed.push(sample),
2876 Err(source) => {
2877 self.exhausted = true;
2878 return Some(Err(LoaderError::Pipeline {
2879 batch: Some(batch),
2880 worker: None,
2881 source: PipelineError::Transform(source),
2882 }));
2883 }
2884 }
2885 self.next_logical_sample += 1;
2886 }
2887 let result =
2888 self.plan
2889 .finish(self.collator, transformed)
2890 .map_err(|source| LoaderError::Pipeline {
2891 batch: Some(batch),
2892 worker: None,
2893 source: PipelineError::Collate(source),
2894 })
2895 .and_then(|value| match self.pin_memory_status {
2896 PinMemoryStatus::Enabled(device) => N::pin(value, device)
2897 .map_err(|source| LoaderError::PinMemory { batch, source }),
2898 PinMemoryStatus::Disabled | PinMemoryStatus::DisabledNoAccelerator => Ok(value),
2899 });
2900 if result.is_err() {
2901 self.exhausted = true;
2902 } else {
2903 self._checkpoint
2904 .committed(self.next_visible, self.next_logical_sample);
2905 self.next_visible += 1;
2906 }
2907 Some(result)
2908 }
2909
2910 fn publish(
2911 &mut self,
2912 worker: usize,
2913 batch: WorkerBatch<
2914 <F::Transform as Transform<D::Sample>>::Output,
2915 <M as MemoryPolicy>::Permit,
2916 >,
2917 ) -> IterResult<D, P, C, F, I> {
2918 debug_assert_eq!(batch.generation, self.generation);
2919 self.outstanding -= 1;
2920 let sequence = batch.batch_sequence;
2921 let WorkerBatch {
2922 batch_sequence,
2923 samples,
2924 permit,
2925 ..
2926 } = batch;
2927 drop(permit);
2928 let result = match catch_unwind(AssertUnwindSafe(|| {
2929 self.plan.finish(self.collator, samples)
2930 })) {
2931 Ok(result) => result.map_err(|source| LoaderError::Pipeline {
2932 batch: Some(sequence),
2933 worker: None,
2934 source: PipelineError::Collate(source),
2935 }),
2936 Err(_) => Err(LoaderError::CoordinatorPanic {
2937 stage: P::FINISH_PANIC_STAGE,
2938 batch: Some(sequence),
2939 }),
2940 };
2941 let result = result.and_then(|value| match self.pin_memory_status {
2942 PinMemoryStatus::Enabled(device) => {
2943 N::pin(value, device).map_err(|source| LoaderError::PinMemory {
2944 batch: batch_sequence,
2945 source,
2946 })
2947 }
2948 PinMemoryStatus::Disabled | PinMemoryStatus::DisabledNoAccelerator => Ok(value),
2949 });
2950 if result.is_ok() {
2951 self._checkpoint
2952 .committed(sequence, self.next_logical_sample);
2953 self.next_visible = self.next_visible.saturating_add(1);
2954 self.fill_available();
2955 } else {
2956 self.stop();
2957 }
2958 let _ = worker;
2959 result
2960 }
2961
2962 fn map_failure(
2963 &mut self,
2964 worker: usize,
2965 batch: Option<u64>,
2966 failure: WorkerStageFailure<D, F, I>,
2967 ) -> IterError<D, P, C, F, I> {
2968 match failure {
2969 WorkerFailure::Dataset(source) => LoaderError::Pipeline {
2970 batch,
2971 worker: Some(worker),
2972 source: PipelineError::Dataset(source),
2973 },
2974 WorkerFailure::Transform(source) => LoaderError::Pipeline {
2975 batch,
2976 worker: Some(worker),
2977 source: PipelineError::Transform(source),
2978 },
2979 WorkerFailure::TransformInit(source) => LoaderError::Pipeline {
2980 batch,
2981 worker: Some(worker),
2982 source: PipelineError::TransformInit(source),
2983 },
2984 WorkerFailure::WorkerInit(source) => LoaderError::Pipeline {
2985 batch,
2986 worker: Some(worker),
2987 source: PipelineError::WorkerInit(source),
2988 },
2989 WorkerFailure::InvalidBatchCardinality { expected, actual } => {
2990 LoaderError::InvalidBatchCardinality {
2991 batch: batch.unwrap_or(self.next_visible),
2992 worker: Some(worker),
2993 expected,
2994 actual,
2995 }
2996 }
2997 WorkerFailure::MemoryLimit { limit, actual } => LoaderError::MemoryLimit {
2998 batch,
2999 worker: Some(worker),
3000 sequence: None,
3001 logical_id: None,
3002 limit,
3003 actual,
3004 },
3005 WorkerFailure::Panic => LoaderError::WorkerPanic { worker, batch },
3006 }
3007 }
3008}
3009
3010impl<D, P, C, F, I, M, N, K> Iterator for WorkerLoaderIter<'_, D, P, C, F, I, M, N, K>
3011where
3012 D: Dataset + Send + Sync + 'static,
3013 D::Sample: Send + 'static,
3014 D::Error: Send + 'static,
3015 F: TransformFactory<D::Sample> + Send + Sync + 'static,
3016 F::Transform: Send + 'static,
3017 <F::Transform as Transform<D::Sample>>::Output: Send + 'static,
3018 <F::Transform as Transform<D::Sample>>::Error: Send + 'static,
3019 F::Error: Send + 'static,
3020 P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
3021 I: WorkerInit + Send + Sync + 'static,
3022 I::Error: Send + 'static,
3023 M: MemoryPolicy + Send + 'static,
3024 N: MapPinPolicy<D, P, C, F>,
3025 K: WorkerBoundaryPolicy<IterError<D, P, C, F, I>>,
3026{
3027 type Item = std::result::Result<
3028 P::Batch,
3029 LoaderError<
3030 PipelineError<
3031 D::Error,
3032 <F::Transform as Transform<D::Sample>>::Error,
3033 P::Error,
3034 F::Error,
3035 I::Error,
3036 >,
3037 >,
3038 >;
3039
3040 fn next(&mut self) -> Option<Self::Item> {
3041 self._checkpoint.begin_next();
3042 if self.exhausted {
3043 return None;
3044 }
3045 if let Some(error) = self.pending_error.take() {
3046 self.stop();
3047 return Some(Err(error));
3048 }
3049 if self.workers == 0 {
3050 return self.next_serial();
3051 }
3052 let mut deadline_armed = false;
3053 loop {
3054 if let Some(error) = self._checkpoint.take_error(self.next_visible) {
3055 self.stop();
3056 return Some(Err(error));
3057 }
3058 if self.ordered
3059 && let Some((worker, batch)) = self.completed.remove(&self.next_visible)
3060 {
3061 if deadline_armed {
3062 self.run_context
3063 .as_ref()
3064 .expect("worker context")
3065 .deadline
3066 .disarm();
3067 }
3068 return Some(self.publish(worker, batch));
3069 }
3070 if self.outstanding == 0 && self.source_exhausted {
3071 if deadline_armed {
3072 self.run_context
3073 .as_ref()
3074 .expect("worker context")
3075 .deadline
3076 .disarm();
3077 }
3078 self.stop();
3079 return None;
3080 }
3081 let run_context = self
3082 .run_context
3083 .as_ref()
3084 .expect("worker iteration has a generation context")
3085 .clone();
3086 if !deadline_armed && let Some(timeout) = self.timeout {
3087 run_context.deadline.arm(timeout);
3088 deadline_armed = true;
3089 }
3090 let remaining = self.timeout.and_then(|_| run_context.deadline.remaining());
3091 let received = self
3092 .pool
3093 .as_ref()
3094 .expect("worker iteration has an active pool")
3095 .pool()
3096 .receive(&run_context, remaining);
3097 let completion = match received {
3098 WorkerReceive::Completion(completion) => completion,
3099 WorkerReceive::Timeout => {
3100 let batch = self.next_visible;
3101 run_context.cancel();
3102 run_context.deadline.disarm();
3103 self.stop();
3104 return Some(Err(LoaderError::Timeout { batch }));
3105 }
3106 WorkerReceive::Cancelled => {
3107 run_context.deadline.disarm();
3108 self.stop();
3109 return Some(Err(LoaderError::Cancelled));
3110 }
3111 WorkerReceive::Closed => {
3112 let batch = self.next_visible;
3113 run_context.deadline.disarm();
3114 self.stop_poisoned();
3115 return Some(Err(LoaderError::ChannelClosed { batch }));
3116 }
3117 };
3118 if completion.generation != self.generation {
3119 continue;
3120 }
3121 match completion.result {
3122 Err(failure) => {
3123 let poisons_pool = matches!(
3124 &failure,
3125 WorkerFailure::TransformInit(_)
3126 | WorkerFailure::WorkerInit(_)
3127 | WorkerFailure::Panic
3128 );
3129 let error =
3130 self.map_failure(completion.worker, completion.batch_sequence, failure);
3131 if !poisons_pool && let Some(sequence) = completion.batch_sequence {
3132 match self._checkpoint.defer_error(sequence, error) {
3133 None => continue,
3134 Some(error) => {
3135 run_context.deadline.disarm();
3136 self.stop();
3137 return Some(Err(error));
3138 }
3139 }
3140 }
3141 run_context.deadline.disarm();
3142 if poisons_pool {
3143 self.stop_poisoned();
3144 } else {
3145 self.stop();
3146 }
3147 return Some(Err(error));
3148 }
3149 Ok(WorkerMessage::Batch(batch))
3150 if self.ordered && batch.batch_sequence != self.next_visible =>
3151 {
3152 self.completed
3153 .insert(batch.batch_sequence, (completion.worker, batch));
3154 }
3155 Ok(WorkerMessage::Batch(batch)) => {
3156 run_context.deadline.disarm();
3157 return Some(self.publish(completion.worker, batch));
3158 }
3159 Ok(WorkerMessage::Quiesced) => continue,
3160 }
3161 }
3162 }
3163}
3164
3165fn validate_configuration(
3166 configuration: &BuilderConfiguration,
3167 explicit: ExplicitArguments,
3168) -> Result<()> {
3169 if configuration.batch_size == 0 {
3170 return Err(invalid_configuration(
3171 "batch_size",
3172 "must be greater than zero",
3173 ));
3174 }
3175 if explicit.sampler && explicit.shuffle {
3176 return Err(invalid_configuration(
3177 "sampler",
3178 "cannot be combined with shuffle",
3179 ));
3180 }
3181 if explicit.batch_sampler
3182 && (explicit.batch_size
3183 || explicit.shuffle
3184 || explicit.sampler
3185 || explicit.drop_last
3186 || explicit.without_batching)
3187 {
3188 return Err(invalid_configuration(
3189 "batch_sampler",
3190 "cannot be combined with batch_size, shuffle, sampler, drop_last, or no batching",
3191 ));
3192 }
3193 if explicit.without_batching && explicit.drop_last && configuration.drop_last {
3194 return Err(invalid_configuration(
3195 "drop_last",
3196 "cannot be enabled when automatic batching is disabled",
3197 ));
3198 }
3199 if configuration.workers == 0 && configuration.timeout.is_some() {
3200 return Err(invalid_configuration(
3201 "timeout",
3202 "requires a positive worker count",
3203 ));
3204 }
3205 if configuration
3206 .timeout
3207 .is_some_and(|timeout| !Deadline::can_represent(timeout))
3208 {
3209 return Err(invalid_configuration(
3210 "timeout",
3211 "exceeds the platform monotonic clock range",
3212 ));
3213 }
3214 if explicit.prefetch_factor && configuration.workers == 0 {
3215 return Err(invalid_configuration(
3216 "prefetch_factor",
3217 "requires a positive worker count",
3218 ));
3219 }
3220 if explicit.prefetch_bytes && configuration.workers == 0 {
3221 return Err(invalid_configuration(
3222 "prefetch_bytes",
3223 "requires a positive worker count",
3224 ));
3225 }
3226 if configuration.prefetch_factor == Some(0) {
3227 return Err(invalid_configuration(
3228 "prefetch_factor",
3229 "must be greater than zero",
3230 ));
3231 }
3232 if configuration.persistent_workers && configuration.workers == 0 {
3233 return Err(invalid_configuration(
3234 "persistent_workers",
3235 "requires a positive worker count",
3236 ));
3237 }
3238 Ok(())
3239}
3240
3241fn validate_exact_builder(
3242 configuration: &BuilderConfiguration,
3243 explicit: ExplicitArguments,
3244 identity: &str,
3245) -> Result<()> {
3246 validate_configuration(configuration, explicit)?;
3247 if identity.is_empty() {
3248 return Err(invalid_configuration(
3249 "dataset_identity",
3250 "must not be empty",
3251 ));
3252 }
3253 if configuration.workers != 0 {
3254 return Err(invalid_configuration(
3255 "workers",
3256 "Task 11 exact checkpoints require serial execution",
3257 ));
3258 }
3259 if !configuration.ordered {
3260 return Err(invalid_configuration(
3261 "in_order",
3262 "exact checkpoints require ordered delivery",
3263 ));
3264 }
3265 Ok(())
3266}
3267
3268fn validate_distributed_rank(identity: &PlanCheckpointIdentity, rank: usize) -> Result<()> {
3269 if let Some(distributed) = identity.distributed
3270 && distributed.rank != rank
3271 {
3272 return Err(invalid_configuration(
3273 "rank",
3274 "loader task rank must equal the distributed sampler rank",
3275 ));
3276 }
3277 Ok(())
3278}
3279
3280fn checkpoint_configuration(
3281 identity: &PlanCheckpointIdentity,
3282 builder: &BuilderConfiguration,
3283 pin_status: PinMemoryStatus,
3284) -> Result<crate::LoaderConfiguration> {
3285 let (replicas, distributed_shuffle, distributed_seed, distributed_drop_last) =
3286 match identity.distributed {
3287 Some(distributed) => (
3288 distributed.replicas,
3289 Some(distributed.shuffle),
3290 Some(distributed.seed),
3291 Some(distributed.drop_last),
3292 ),
3293 None => (1, None, None, None),
3294 };
3295 let pin_request = checkpoint_pin_request(builder.pin_memory)?;
3296 let pin_status = checkpoint_pin_status(pin_status)?;
3297 Ok(crate::LoaderConfiguration {
3298 batch_size: identity.batch_size,
3299 drop_last: identity.drop_last,
3300 workers: builder.workers,
3301 prefetch_factor: builder.prefetch_factor,
3302 in_order: true,
3303 loader_seed: builder.loader_seed,
3304 rank: builder.rank,
3305 replicas,
3306 distributed_shuffle,
3307 distributed_seed,
3308 distributed_drop_last,
3309 sampler_kind: identity.sampler_kind.clone(),
3310 pin_request,
3311 pin_status,
3312 })
3313}
3314
3315fn checkpoint_pin_request(request: PinRequest) -> Result<CheckpointPinRequest> {
3316 Ok(match request {
3317 PinRequest::Disabled => CheckpointPinRequest::Disabled,
3318 PinRequest::Auto => CheckpointPinRequest::Auto,
3319 PinRequest::Explicit(Device::Cuda(index)) => CheckpointPinRequest::ExplicitCuda { index },
3320 PinRequest::Explicit(device) => {
3321 return Err(invalid_configuration(
3322 "pin_memory",
3323 format!("unsupported checkpoint pin device {device:?}"),
3324 ));
3325 }
3326 })
3327}
3328
3329fn checkpoint_pin_status(status: PinMemoryStatus) -> Result<CheckpointPinStatus> {
3330 Ok(match status {
3331 PinMemoryStatus::Disabled => CheckpointPinStatus::Disabled,
3332 PinMemoryStatus::DisabledNoAccelerator => CheckpointPinStatus::DisabledNoAccelerator,
3333 PinMemoryStatus::Enabled(Device::Cuda(index)) => CheckpointPinStatus::Cuda { index },
3334 PinMemoryStatus::Enabled(device) => {
3335 return Err(invalid_configuration(
3336 "pin_memory",
3337 format!("unsupported effective checkpoint pin device {device:?}"),
3338 ));
3339 }
3340 })
3341}
3342
3343fn validate_static_loader_state_envelope<D, S, T, C>(
3344 state: &LoaderState<D, S, T, C>,
3345 identity: &str,
3346 builder: &BuilderConfiguration,
3347 pin_status: PinMemoryStatus,
3348 automatic_batching: bool,
3349 expected_generation: u64,
3350) -> Result<()> {
3351 let expected_batch_size = automatic_batching.then_some(builder.batch_size);
3352 let expected_drop_last = automatic_batching && builder.drop_last;
3353 let expected_pin_request = checkpoint_pin_request(builder.pin_memory)?;
3354 let expected_pin_status = checkpoint_pin_status(pin_status)?;
3355
3356 require_checkpoint_equal(
3357 "schema_version",
3358 state.schema_version,
3359 LOADER_STATE_SCHEMA_VERSION,
3360 )?;
3361 require_checkpoint_equal(
3362 "dataset_identity",
3363 state.dataset_identity.as_str(),
3364 identity,
3365 )?;
3366 require_checkpoint_equal(
3367 "iterator_generation",
3368 state.iterator_generation,
3369 expected_generation,
3370 )?;
3371 require_checkpoint_equal(
3372 "rng_derivation_version",
3373 state.rng_derivation_version,
3374 TASK_RNG_DERIVATION_VERSION,
3375 )?;
3376 require_checkpoint_equal(
3377 "worker_seed_derivation_version",
3378 state.worker_seed_derivation_version,
3379 WORKER_SEED_DERIVATION_VERSION,
3380 )?;
3381 require_checkpoint_equal(
3382 "batch_size",
3383 state.configuration.batch_size,
3384 expected_batch_size,
3385 )?;
3386 require_checkpoint_equal(
3387 "drop_last",
3388 state.configuration.drop_last,
3389 expected_drop_last,
3390 )?;
3391 require_checkpoint_equal("workers", state.configuration.workers, builder.workers)?;
3392 require_checkpoint_equal(
3393 "prefetch_factor",
3394 state.configuration.prefetch_factor,
3395 builder.prefetch_factor,
3396 )?;
3397 require_checkpoint_equal("in_order", state.configuration.in_order, builder.ordered)?;
3398 require_checkpoint_equal(
3399 "loader_seed",
3400 state.configuration.loader_seed,
3401 builder.loader_seed,
3402 )?;
3403 require_checkpoint_equal("rank", state.configuration.rank, builder.rank)?;
3404 require_checkpoint_equal(
3405 "pin_request",
3406 &state.configuration.pin_request,
3407 &expected_pin_request,
3408 )?;
3409 require_checkpoint_equal(
3410 "pin_status",
3411 &state.configuration.pin_status,
3412 &expected_pin_status,
3413 )
3414}
3415
3416fn validate_plan_loader_state_envelope<D, S, T, C>(
3417 state: &LoaderState<D, S, T, C>,
3418 expected: &crate::LoaderConfiguration,
3419) -> Result<()> {
3420 require_checkpoint_equal("replicas", state.configuration.replicas, expected.replicas)?;
3421 require_checkpoint_equal(
3422 "distributed_shuffle",
3423 state.configuration.distributed_shuffle,
3424 expected.distributed_shuffle,
3425 )?;
3426 require_checkpoint_equal(
3427 "distributed_seed",
3428 state.configuration.distributed_seed,
3429 expected.distributed_seed,
3430 )?;
3431 require_checkpoint_equal(
3432 "distributed_drop_last",
3433 state.configuration.distributed_drop_last,
3434 expected.distributed_drop_last,
3435 )?;
3436 require_checkpoint_equal(
3437 "sampler_kind",
3438 state.configuration.sampler_kind.as_str(),
3439 expected.sampler_kind.as_str(),
3440 )
3441}
3442
3443fn require_checkpoint_equal<T>(field: &'static str, actual: T, expected: T) -> Result<()>
3444where
3445 T: PartialEq,
3446{
3447 if actual != expected {
3448 return Err(invalid_configuration(
3449 field,
3450 "checkpoint value does not match the loader",
3451 ));
3452 }
3453 Ok(())
3454}
3455
3456fn resolve_pin_request(request: PinRequest) -> Result<PinMemoryStatus> {
3457 match request {
3458 PinRequest::Disabled => Ok(PinMemoryStatus::Disabled),
3459 PinRequest::Auto => {
3460 let capabilities = available_devices();
3461 if capabilities.cuda && capabilities.cuda_device_count > 0 {
3462 Ok(PinMemoryStatus::Enabled(Device::Cuda(0)))
3463 } else {
3464 Ok(PinMemoryStatus::DisabledNoAccelerator)
3465 }
3466 }
3467 PinRequest::Explicit(Device::Cuda(index)) => {
3468 let capabilities = available_devices();
3469 if capabilities.cuda && index < capabilities.cuda_device_count {
3470 Ok(PinMemoryStatus::Enabled(Device::Cuda(index)))
3471 } else {
3472 Err(invalid_configuration(
3473 "pin_memory",
3474 format!(
3475 "CUDA device {index} was requested, but the linked runtime exposes {} available CUDA device(s)",
3476 capabilities.cuda_device_count
3477 ),
3478 ))
3479 }
3480 }
3481 PinRequest::Explicit(device) => Err(invalid_configuration(
3482 "pin_memory",
3483 format!("only an available CUDA device can back pinned host memory, got {device:?}"),
3484 )),
3485 }
3486}
3487
3488fn validate_auto_batch_boundary(
3489 exact_len: Option<usize>,
3490 batch_size: NonZeroUsize,
3491 drop_last: bool,
3492 next_batch: u64,
3493 next_logical_sample: u64,
3494) -> Result<()> {
3495 let batch_size = u64::try_from(batch_size.get()).map_err(|_| {
3496 invalid_configuration(
3497 "checkpoint",
3498 "batch size does not fit the checkpoint schema",
3499 )
3500 })?;
3501 let complete = next_logical_sample / batch_size;
3502 let aligned = next_logical_sample.is_multiple_of(batch_size);
3503 let final_short = match exact_len {
3504 Some(length) => {
3505 let length = u64::try_from(length).map_err(|_| {
3506 invalid_configuration(
3507 "checkpoint",
3508 "sampler length does not fit the checkpoint schema",
3509 )
3510 })?;
3511 !drop_last
3512 && next_logical_sample == length
3513 && !aligned
3514 && next_batch == complete.saturating_add(1)
3515 }
3516 None => false,
3517 };
3518 if (!aligned || next_batch != complete) && !final_short {
3519 return Err(invalid_configuration(
3520 "checkpoint",
3521 "batch and logical sample cursors are not on a visible batch boundary",
3522 ));
3523 }
3524 if let Some(length) = exact_len {
3525 let length = u64::try_from(length).map_err(|_| {
3526 invalid_configuration(
3527 "checkpoint",
3528 "sampler length does not fit the checkpoint schema",
3529 )
3530 })?;
3531 if next_logical_sample > length {
3532 return Err(invalid_configuration(
3533 "checkpoint",
3534 "logical sample cursor exceeds the sampler length",
3535 ));
3536 }
3537 }
3538 Ok(())
3539}
3540
3541fn batch_count(length: usize, batch_size: NonZeroUsize, drop_last: bool) -> usize {
3542 let batch_size = batch_size.get();
3543 let complete = length / batch_size;
3544 complete + usize::from(!drop_last && !length.is_multiple_of(batch_size))
3545}
3546
3547fn invalid_configuration(field: &'static str, reason: impl Into<String>) -> RustTorchError {
3548 RustTorchError::InvalidConfiguration {
3549 field,
3550 reason: reason.into(),
3551 }
3552}