pub struct StreamDataLoader<S, C, F = IdentityTransformFactory, I = NoWorkerInit, M = MemoryDisabled, N = PinDisabled>where
S: WorkerSourceFactory,
F: TransformFactory<S::Sample>,
C: Collate<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output>,
I: WorkerInit,
M: StreamMemoryPolicy,{ /* private fields */ }Expand description
Reusable owner of a sharded input pipeline.
Create it with StreamDataLoaderBuilder, whose example reads lazy disjoint
shards into global batches. Call Self::iter for each pass and
Self::set_epoch before the next epoch when transforms use task randomness.
Self::len is the global batch count when the factory knows its exact length.
Each pass creates new source iterators. Persistent mode keeps worker threads
and transforms across passes; dropping this owner shuts them down and joins
them. Use ExactStreamDataLoader when you need to retain a source cursor
across application restarts.
Implementations§
Source§impl<S, C, F, I, M, N> StreamDataLoader<S, C, F, I, M, N>where
S: WorkerSourceFactory,
F: TransformFactory<S::Sample>,
C: Collate<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output>,
I: WorkerInit,
M: StreamMemoryPolicy,
impl<S, C, F, I, M, N> StreamDataLoader<S, C, F, I, M, N>where
S: WorkerSourceFactory,
F: TransformFactory<S::Sample>,
C: Collate<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output>,
I: WorkerInit,
M: StreamMemoryPolicy,
Sourcepub fn len(&self) -> Option<usize>
pub fn len(&self) -> Option<usize>
Returns the exact global batch count when the factory is sized.
Sourcepub fn effective_prefetch_factor(&self) -> NonZeroUsize
pub fn effective_prefetch_factor(&self) -> NonZeroUsize
Returns the effective records-prefetched-per-worker factor.
Sourcepub fn effective_prefetch_bytes(&self) -> Option<NonZeroUsize>
pub fn effective_prefetch_bytes(&self) -> Option<NonZeroUsize>
Returns the effective post-transform record byte budget.
Sourcepub fn pin_memory_enabled(&self) -> bool
pub fn pin_memory_enabled(&self) -> bool
Returns whether recursive batch pinning was requested.
Sourcepub fn pin_memory_status(&self) -> PinMemoryStatus
pub fn pin_memory_status(&self) -> PinMemoryStatus
Returns the effective recursive pinning behavior.
Sourcepub fn is_ordered(&self) -> bool
pub fn is_ordered(&self) -> bool
Returns whether ordered delivery is enabled.
Source§impl<S, C, F, I, M, N> StreamDataLoader<S, C, F, I, M, N>where
S: WorkerSourceFactory,
F: TransformFactory<S::Sample> + Send + Sync + 'static,
F::Transform: Send + 'static,
<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output: Send + 'static,
<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Error: Send + 'static,
F::Error: Send + 'static,
C: Collate<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output>,
I: WorkerInit + Send + Sync + 'static,
I::Error: Send + 'static,
M: StreamMemoryPolicy + Send + 'static,
N: StreamPinPolicy<S, C, F>,
impl<S, C, F, I, M, N> StreamDataLoader<S, C, F, I, M, N>where
S: WorkerSourceFactory,
F: TransformFactory<S::Sample> + Send + Sync + 'static,
F::Transform: Send + 'static,
<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output: Send + 'static,
<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Error: Send + 'static,
F::Error: Send + 'static,
C: Collate<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output>,
I: WorkerInit + Send + Sync + 'static,
I::Error: Send + 'static,
M: StreamMemoryPolicy + Send + 'static,
N: StreamPinPolicy<S, C, F>,
Sourcepub fn iter(&mut self) -> StreamLoaderIter<'_, S, C, F, I, M, N> ⓘ
pub fn iter(&mut self) -> StreamLoaderIter<'_, S, C, F, I, M, N> ⓘ
Starts a fresh explicitly sharded worker generation.