pub struct OwnedDataLoader<D, P, C, F = IdentityTransformFactory, I = NoWorkerInit, X = SerialExecution, M = MemoryDisabled, N = PinDisabled, K = CheckpointDisabled>{ /* private fields */ }Expand description
A reusable dataset pipeline, created by crate::DataLoader::builder.
Call iter() for a pass over the selected epoch and set_epoch(epoch) before
the next epoch’s shuffle. The iterator borrows this owner mutably, preventing
two overlapping passes over the same pipeline. Self::len reports batches
(or individual outputs without batching), not the dataset’s sample count.
The DataLoaderBuilder examples cover worker prefetch, custom padding and
inference without batching. Serial loading accepts local non-Send types;
worker loading shares a thread-safe dataset and fetches samples concurrently.
Dropping an iteration cancels its outstanding work and waits for it to stop.
Persistent workers remain alive for reuse until this owner is dropped.
Exact loaders additionally expose checkpoint() on their iterator. Save
after a successful batch and recreate the matching builder to resume;
crate::LoaderState demonstrates the complete sequence.
Implementations§
Source§impl<D, P, C, F, N> OwnedDataLoader<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, WorkerCheckpointActive<<P as LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>>::Iter, <F as TransformFactory<<D as Dataset>::Sample>>::Transform, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State>>where
D: WorkerReplayDataset + Send + Sync + 'static,
D::Sample: Send + 'static,
D::Error: Send + 'static,
F: TransformFactory<D::Sample> + Send + Sync + 'static,
F::Transform: WorkerCheckpoint + Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State: Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output: Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Error: Send + 'static,
F::Error: Send + 'static,
P: CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>,
N: MapPinPolicy<D, P, C, F>,
impl<D, P, C, F, N> OwnedDataLoader<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, WorkerCheckpointActive<<P as LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>>::Iter, <F as TransformFactory<<D as Dataset>::Sample>>::Transform, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State>>where
D: WorkerReplayDataset + Send + Sync + 'static,
D::Sample: Send + 'static,
D::Error: Send + 'static,
F: TransformFactory<D::Sample> + Send + Sync + 'static,
F::Transform: WorkerCheckpoint + Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State: Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output: Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Error: Send + 'static,
F::Error: Send + 'static,
P: CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>,
N: MapPinPolicy<D, P, C, F>,
Sourcepub fn iter(
&mut self,
) -> WorkerLoaderIter<'_, D, P, C, F, NoWorkerInit, MemoryDisabled, N, WorkerCheckpointIteration<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State, LoaderError<PipelineError<<D as Dataset>::Error, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Error, <P as LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>>::Error, <F as TransformFactory<<D as Dataset>::Sample>>::Error, <NoWorkerInit as WorkerInit>::Error>>>> ⓘ
pub fn iter( &mut self, ) -> WorkerLoaderIter<'_, D, P, C, F, NoWorkerInit, MemoryDisabled, N, WorkerCheckpointIteration<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State, LoaderError<PipelineError<<D as Dataset>::Error, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Error, <P as LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>>::Error, <F as TransformFactory<<D as Dataset>::Sample>>::Error, <NoWorkerInit as WorkerInit>::Error>>>> ⓘ
Starts the retained resume once, then fresh deterministic worker generations.
Source§impl<D, P, C, F, I, M, N> OwnedDataLoader<D, P, C, F, I, SerialExecution, M, N, CheckpointDisabled>where
D: Dataset,
F: TransformFactory<D::Sample>,
P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
I: WorkerInit,
M: MemoryPolicy,
N: MapPinPolicy<D, P, C, F>,
impl<D, P, C, F, I, M, N> OwnedDataLoader<D, P, C, F, I, SerialExecution, M, N, CheckpointDisabled>where
D: Dataset,
F: TransformFactory<D::Sample>,
P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
I: WorkerInit,
M: MemoryPolicy,
N: MapPinPolicy<D, P, C, F>,
Sourcepub fn iter(
&mut self,
) -> LoaderIter<'_, D, P, C, F, I, M, N, CheckpointDisabled> ⓘ
pub fn iter( &mut self, ) -> LoaderIter<'_, D, P, C, F, I, M, N, CheckpointDisabled> ⓘ
Starts a fresh finite iteration for the configured epoch.
Source§impl<D, P, C, F, I, N> OwnedDataLoader<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointActive<P::Iter, F::Transform>>where
D: DatasetCheckpoint,
F: TransformFactory<D::Sample>,
F::Transform: WorkerCheckpoint,
P: LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C> + CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>,
I: WorkerInit,
N: MapPinPolicy<D, P, C, F>,
impl<D, P, C, F, I, N> OwnedDataLoader<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointActive<P::Iter, F::Transform>>where
D: DatasetCheckpoint,
F: TransformFactory<D::Sample>,
F::Transform: WorkerCheckpoint,
P: LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C> + CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>,
I: WorkerInit,
N: MapPinPolicy<D, P, C, F>,
Sourcepub fn iter(
&mut self,
) -> LoaderIter<'_, D, P, C, F, I, MemoryDisabled, N, CheckpointIteration> ⓘ
pub fn iter( &mut self, ) -> LoaderIter<'_, D, P, C, F, I, MemoryDisabled, N, CheckpointIteration> ⓘ
Starts the retained resumed cursor once, then fresh exact iterations.
Sourcepub fn set_epoch(&mut self, epoch: u64)
pub fn set_epoch(&mut self, epoch: u64)
Selects the epoch used by future iterations.
Calling this before the retained resumed cursor is consumed deliberately discards that cursor and its restored transform, so the selected epoch starts as an ordinary fresh iteration instead of mixing two epochs.
Source§impl<D, P, C, F, I, X, M, N> OwnedDataLoader<D, P, C, F, I, X, M, N, CheckpointDisabled>where
D: Dataset,
F: TransformFactory<D::Sample>,
P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
I: WorkerInit,
M: MemoryPolicy,
impl<D, P, C, F, I, X, M, N> OwnedDataLoader<D, P, C, F, I, X, M, N, CheckpointDisabled>where
D: Dataset,
F: TransformFactory<D::Sample>,
P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
I: WorkerInit,
M: MemoryPolicy,
Source§impl<D, P, C, F, I, X, M, N, K> OwnedDataLoader<D, P, C, F, I, X, M, N, K>where
D: Dataset,
F: TransformFactory<D::Sample>,
P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
I: WorkerInit,
M: MemoryPolicy,
impl<D, P, C, F, I, X, M, N, K> OwnedDataLoader<D, P, C, F, I, X, M, N, K>where
D: Dataset,
F: TransformFactory<D::Sample>,
P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
I: WorkerInit,
M: MemoryPolicy,
Sourcepub fn len(&self) -> Option<usize>
pub fn len(&self) -> Option<usize>
Returns the exact number of yielded items when the source is sized.
Sourcepub fn is_empty(&self) -> Option<bool>
pub fn is_empty(&self) -> Option<bool>
Returns whether iteration is empty when the source is sized.
Sourcepub fn effective_prefetch_factor(&self) -> Option<NonZeroUsize>
pub fn effective_prefetch_factor(&self) -> Option<NonZeroUsize>
Returns the effective batches-prefetched-per-worker value.
Sourcepub fn is_ordered(&self) -> bool
pub fn is_ordered(&self) -> bool
Returns whether results are configured for sampler order.
Sourcepub fn pin_memory_enabled(&self) -> bool
pub fn pin_memory_enabled(&self) -> bool
Returns whether pinning was requested.
Sourcepub fn pin_memory_status(&self) -> PinMemoryStatus
pub fn pin_memory_status(&self) -> PinMemoryStatus
Returns the effective recursive pinning behavior.
Sourcepub fn effective_prefetch_bytes(&self) -> Option<NonZeroUsize>
pub fn effective_prefetch_bytes(&self) -> Option<NonZeroUsize>
Returns the effective post-transform prefetch byte budget.
Sourcepub fn persistent_workers_enabled(&self) -> bool
pub fn persistent_workers_enabled(&self) -> bool
Returns whether persistent workers were requested.
Source§impl<D, P, C, F, I, M, N> OwnedDataLoader<D, P, C, F, I, WorkerExecution, M, N, CheckpointDisabled>where
D: Dataset + Send + Sync + 'static,
D::Sample: Send + 'static,
D::Error: Send + 'static,
F: TransformFactory<D::Sample> + Send + Sync + 'static,
F::Transform: Send + 'static,
<F::Transform as Transform<D::Sample>>::Output: Send + 'static,
<F::Transform as Transform<D::Sample>>::Error: Send + 'static,
F::Error: Send + 'static,
P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
I: WorkerInit + Send + Sync + 'static,
I::Error: Send + 'static,
M: MemoryPolicy + Send + 'static,
N: MapPinPolicy<D, P, C, F>,
impl<D, P, C, F, I, M, N> OwnedDataLoader<D, P, C, F, I, WorkerExecution, M, N, CheckpointDisabled>where
D: Dataset + Send + Sync + 'static,
D::Sample: Send + 'static,
D::Error: Send + 'static,
F: TransformFactory<D::Sample> + Send + Sync + 'static,
F::Transform: Send + 'static,
<F::Transform as Transform<D::Sample>>::Output: Send + 'static,
<F::Transform as Transform<D::Sample>>::Error: Send + 'static,
F::Error: Send + 'static,
P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
I: WorkerInit + Send + Sync + 'static,
I::Error: Send + 'static,
M: MemoryPolicy + Send + 'static,
N: MapPinPolicy<D, P, C, F>,
Sourcepub fn iter(
&mut self,
) -> WorkerLoaderIter<'_, D, P, C, F, I, M, N, CheckpointDisabled> ⓘ
pub fn iter( &mut self, ) -> WorkerLoaderIter<'_, D, P, C, F, I, M, N, CheckpointDisabled> ⓘ
Starts a fresh generation on a bounded worker pool, or the explicit zero-worker path.