pub struct OwnedDataLoader<D, P, C, F = IdentityTransformFactory, I = NoWorkerInit, X = SerialExecution, M = MemoryDisabled, N = PinDisabled, K = CheckpointDisabled>{ /* private fields */ }Expand description
An owned, re-iterable map-style data loader.
The default SerialExecution capability accepts local non-Send data.
Calling DataLoaderBuilder::workers selects WorkerExecution, where
positive counts share the dataset through Arc. Each worker owns a
bounded task lane and one transform instance; the coordinator alone owns
ordering and collation. This differs from PyTorch’s process-local dataset
copies and worker-side collation while preserving bounded prefetch,
deterministic routing, task-local randomness, and ordered delivery.
Implementations§
Source§impl<D, P, C, F, N> OwnedDataLoader<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, WorkerCheckpointActive<<P as LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>>::Iter, <F as TransformFactory<<D as Dataset>::Sample>>::Transform, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State>>where
D: WorkerReplayDataset + Send + Sync + 'static,
D::Sample: Send + 'static,
D::Error: Send + 'static,
F: TransformFactory<D::Sample> + Send + Sync + 'static,
F::Transform: WorkerCheckpoint + Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State: Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output: Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Error: Send + 'static,
F::Error: Send + 'static,
P: CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>,
N: MapPinPolicy<D, P, C, F>,
impl<D, P, C, F, N> OwnedDataLoader<D, P, C, F, NoWorkerInit, WorkerExecution, MemoryDisabled, N, WorkerCheckpointActive<<P as LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>>::Iter, <F as TransformFactory<<D as Dataset>::Sample>>::Transform, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State>>where
D: WorkerReplayDataset + Send + Sync + 'static,
D::Sample: Send + 'static,
D::Error: Send + 'static,
F: TransformFactory<D::Sample> + Send + Sync + 'static,
F::Transform: WorkerCheckpoint + Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State: Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output: Send + 'static,
<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Error: Send + 'static,
F::Error: Send + 'static,
P: CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>,
N: MapPinPolicy<D, P, C, F>,
Sourcepub fn iter(
&mut self,
) -> WorkerLoaderIter<'_, D, P, C, F, NoWorkerInit, MemoryDisabled, N, WorkerCheckpointIteration<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State, LoaderError<PipelineError<<D as Dataset>::Error, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Error, <P as LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>>::Error, <F as TransformFactory<<D as Dataset>::Sample>>::Error, <NoWorkerInit as WorkerInit>::Error>>>> ⓘ
pub fn iter( &mut self, ) -> WorkerLoaderIter<'_, D, P, C, F, NoWorkerInit, MemoryDisabled, N, WorkerCheckpointIteration<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as WorkerCheckpoint>::State, LoaderError<PipelineError<<D as Dataset>::Error, <<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Error, <P as LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>>::Error, <F as TransformFactory<<D as Dataset>::Sample>>::Error, <NoWorkerInit as WorkerInit>::Error>>>> ⓘ
Starts the retained resume once, then fresh deterministic worker generations.
Source§impl<D, P, C, F, I, M, N> OwnedDataLoader<D, P, C, F, I, SerialExecution, M, N, CheckpointDisabled>where
D: Dataset,
F: TransformFactory<D::Sample>,
P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
I: WorkerInit,
M: MemoryPolicy,
N: MapPinPolicy<D, P, C, F>,
impl<D, P, C, F, I, M, N> OwnedDataLoader<D, P, C, F, I, SerialExecution, M, N, CheckpointDisabled>where
D: Dataset,
F: TransformFactory<D::Sample>,
P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
I: WorkerInit,
M: MemoryPolicy,
N: MapPinPolicy<D, P, C, F>,
Sourcepub fn iter(
&mut self,
) -> LoaderIter<'_, D, P, C, F, I, M, N, CheckpointDisabled> ⓘ
pub fn iter( &mut self, ) -> LoaderIter<'_, D, P, C, F, I, M, N, CheckpointDisabled> ⓘ
Starts a fresh finite iteration for the configured epoch.
Source§impl<D, P, C, F, I, N> OwnedDataLoader<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointActive<P::Iter, F::Transform>>where
D: DatasetCheckpoint,
F: TransformFactory<D::Sample>,
F::Transform: WorkerCheckpoint,
P: LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C> + CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>,
I: WorkerInit,
N: MapPinPolicy<D, P, C, F>,
impl<D, P, C, F, I, N> OwnedDataLoader<D, P, C, F, I, SerialExecution, MemoryDisabled, N, CheckpointActive<P::Iter, F::Transform>>where
D: DatasetCheckpoint,
F: TransformFactory<D::Sample>,
F::Transform: WorkerCheckpoint,
P: LoaderPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C> + CheckpointPlan<<<F as TransformFactory<<D as Dataset>::Sample>>::Transform as Transform<<D as Dataset>::Sample>>::Output, C>,
I: WorkerInit,
N: MapPinPolicy<D, P, C, F>,
Sourcepub fn iter(
&mut self,
) -> LoaderIter<'_, D, P, C, F, I, MemoryDisabled, N, CheckpointIteration> ⓘ
pub fn iter( &mut self, ) -> LoaderIter<'_, D, P, C, F, I, MemoryDisabled, N, CheckpointIteration> ⓘ
Starts the retained resumed cursor once, then fresh exact iterations.
Sourcepub fn set_epoch(&mut self, epoch: u64)
pub fn set_epoch(&mut self, epoch: u64)
Selects the epoch used by future iterations.
Calling this before the retained resumed cursor is consumed deliberately discards that cursor and its restored transform, so the selected epoch starts as an ordinary fresh iteration instead of mixing two epochs.
Source§impl<D, P, C, F, I, X, M, N> OwnedDataLoader<D, P, C, F, I, X, M, N, CheckpointDisabled>where
D: Dataset,
F: TransformFactory<D::Sample>,
P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
I: WorkerInit,
M: MemoryPolicy,
impl<D, P, C, F, I, X, M, N> OwnedDataLoader<D, P, C, F, I, X, M, N, CheckpointDisabled>where
D: Dataset,
F: TransformFactory<D::Sample>,
P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
I: WorkerInit,
M: MemoryPolicy,
Source§impl<D, P, C, F, I, X, M, N, K> OwnedDataLoader<D, P, C, F, I, X, M, N, K>where
D: Dataset,
F: TransformFactory<D::Sample>,
P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
I: WorkerInit,
M: MemoryPolicy,
impl<D, P, C, F, I, X, M, N, K> OwnedDataLoader<D, P, C, F, I, X, M, N, K>where
D: Dataset,
F: TransformFactory<D::Sample>,
P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
I: WorkerInit,
M: MemoryPolicy,
Sourcepub fn len(&self) -> Option<usize>
pub fn len(&self) -> Option<usize>
Returns the exact number of yielded items when the source is sized.
Sourcepub fn is_empty(&self) -> Option<bool>
pub fn is_empty(&self) -> Option<bool>
Returns whether iteration is empty when the source is sized.
Sourcepub fn effective_prefetch_factor(&self) -> Option<NonZeroUsize>
pub fn effective_prefetch_factor(&self) -> Option<NonZeroUsize>
Returns the effective batches-prefetched-per-worker value.
Sourcepub fn is_ordered(&self) -> bool
pub fn is_ordered(&self) -> bool
Returns whether results are configured for sampler order.
Sourcepub fn pin_memory_enabled(&self) -> bool
pub fn pin_memory_enabled(&self) -> bool
Returns whether pinning was requested.
Sourcepub fn pin_memory_status(&self) -> PinMemoryStatus
pub fn pin_memory_status(&self) -> PinMemoryStatus
Returns the effective recursive pinning behavior.
Sourcepub fn effective_prefetch_bytes(&self) -> Option<NonZeroUsize>
pub fn effective_prefetch_bytes(&self) -> Option<NonZeroUsize>
Returns the effective post-transform prefetch byte budget.
Sourcepub fn persistent_workers_enabled(&self) -> bool
pub fn persistent_workers_enabled(&self) -> bool
Returns whether persistent workers were requested.
Source§impl<D, P, C, F, I, M, N> OwnedDataLoader<D, P, C, F, I, WorkerExecution, M, N, CheckpointDisabled>where
D: Dataset + Send + Sync + 'static,
D::Sample: Send + 'static,
D::Error: Send + 'static,
F: TransformFactory<D::Sample> + Send + Sync + 'static,
F::Transform: Send + 'static,
<F::Transform as Transform<D::Sample>>::Output: Send + 'static,
<F::Transform as Transform<D::Sample>>::Error: Send + 'static,
F::Error: Send + 'static,
P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
I: WorkerInit + Send + Sync + 'static,
I::Error: Send + 'static,
M: MemoryPolicy + Send + 'static,
N: MapPinPolicy<D, P, C, F>,
impl<D, P, C, F, I, M, N> OwnedDataLoader<D, P, C, F, I, WorkerExecution, M, N, CheckpointDisabled>where
D: Dataset + Send + Sync + 'static,
D::Sample: Send + 'static,
D::Error: Send + 'static,
F: TransformFactory<D::Sample> + Send + Sync + 'static,
F::Transform: Send + 'static,
<F::Transform as Transform<D::Sample>>::Output: Send + 'static,
<F::Transform as Transform<D::Sample>>::Error: Send + 'static,
F::Error: Send + 'static,
P: LoaderPlan<<F::Transform as Transform<D::Sample>>::Output, C>,
I: WorkerInit + Send + Sync + 'static,
I::Error: Send + 'static,
M: MemoryPolicy + Send + 'static,
N: MapPinPolicy<D, P, C, F>,
Sourcepub fn iter(
&mut self,
) -> WorkerLoaderIter<'_, D, P, C, F, I, M, N, CheckpointDisabled> ⓘ
pub fn iter( &mut self, ) -> WorkerLoaderIter<'_, D, P, C, F, I, M, N, CheckpointDisabled> ⓘ
Starts a fresh generation on a bounded worker pool, or the explicit zero-worker path.