pub struct StreamDataLoader<S, C, F = IdentityTransformFactory, I = NoWorkerInit, M = MemoryDisabled, N = PinDisabled>where
S: WorkerSourceFactory,
F: TransformFactory<S::Sample>,
C: Collate<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output>,
I: WorkerInit,
M: StreamMemoryPolicy,{ /* private fields */ }Expand description
Owned, re-iterable explicitly sharded stream loader.
Implementations§
Source§impl<S, C, F, I, M, N> StreamDataLoader<S, C, F, I, M, N>where
S: WorkerSourceFactory,
F: TransformFactory<S::Sample>,
C: Collate<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output>,
I: WorkerInit,
M: StreamMemoryPolicy,
impl<S, C, F, I, M, N> StreamDataLoader<S, C, F, I, M, N>where
S: WorkerSourceFactory,
F: TransformFactory<S::Sample>,
C: Collate<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output>,
I: WorkerInit,
M: StreamMemoryPolicy,
Sourcepub fn len(&self) -> Option<usize>
pub fn len(&self) -> Option<usize>
Returns the exact global batch count when the factory is sized.
Sourcepub fn effective_prefetch_factor(&self) -> NonZeroUsize
pub fn effective_prefetch_factor(&self) -> NonZeroUsize
Returns the effective records-prefetched-per-worker factor.
Sourcepub fn effective_prefetch_bytes(&self) -> Option<NonZeroUsize>
pub fn effective_prefetch_bytes(&self) -> Option<NonZeroUsize>
Returns the effective post-transform record byte budget.
Sourcepub fn pin_memory_enabled(&self) -> bool
pub fn pin_memory_enabled(&self) -> bool
Returns whether recursive batch pinning was requested.
Sourcepub fn pin_memory_status(&self) -> PinMemoryStatus
pub fn pin_memory_status(&self) -> PinMemoryStatus
Returns the effective recursive pinning behavior.
Sourcepub fn is_ordered(&self) -> bool
pub fn is_ordered(&self) -> bool
Returns whether ordered delivery is enabled.
Source§impl<S, C, F, I, M, N> StreamDataLoader<S, C, F, I, M, N>where
S: WorkerSourceFactory,
F: TransformFactory<S::Sample> + Send + Sync + 'static,
F::Transform: Send + 'static,
<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output: Send + 'static,
<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Error: Send + 'static,
F::Error: Send + 'static,
C: Collate<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output>,
I: WorkerInit + Send + Sync + 'static,
I::Error: Send + 'static,
M: StreamMemoryPolicy + Send + 'static,
N: StreamPinPolicy<S, C, F>,
impl<S, C, F, I, M, N> StreamDataLoader<S, C, F, I, M, N>where
S: WorkerSourceFactory,
F: TransformFactory<S::Sample> + Send + Sync + 'static,
F::Transform: Send + 'static,
<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output: Send + 'static,
<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Error: Send + 'static,
F::Error: Send + 'static,
C: Collate<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output>,
I: WorkerInit + Send + Sync + 'static,
I::Error: Send + 'static,
M: StreamMemoryPolicy + Send + 'static,
N: StreamPinPolicy<S, C, F>,
Sourcepub fn iter(&mut self) -> StreamLoaderIter<'_, S, C, F, I, M, N> ⓘ
pub fn iter(&mut self) -> StreamLoaderIter<'_, S, C, F, I, M, N> ⓘ
Starts a fresh explicitly sharded worker generation.
Auto Trait Implementations§
impl<S, C, F = IdentityTransformFactory, I = NoWorkerInit, M = MemoryDisabled, N = PinDisabled> !Freeze for StreamDataLoader<S, C, F, I, M, N>
impl<S, C, F = IdentityTransformFactory, I = NoWorkerInit, M = MemoryDisabled, N = PinDisabled> !RefUnwindSafe for StreamDataLoader<S, C, F, I, M, N>
impl<S, C, F = IdentityTransformFactory, I = NoWorkerInit, M = MemoryDisabled, N = PinDisabled> !UnwindSafe for StreamDataLoader<S, C, F, I, M, N>
impl<S, C, F, I, M, N> Send for StreamDataLoader<S, C, F, I, M, N>where
Arc<S>: Send,
C: Send,
Arc<F>: Send,
Arc<I>: Send,
Option<StreamWorkerPool<S, F, I, M>>: Send,
Mutex<Vec<(u64, BufferedRecord<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output, <M as MemoryPolicy>::Permit>)>>: Send,
Option<fn(&<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output) -> usize>: Send,
PhantomData<(M, N)>: Send,
impl<S, C, F, I, M, N> Sync for StreamDataLoader<S, C, F, I, M, N>where
Arc<S>: Sync,
C: Sync,
Arc<F>: Sync,
Arc<I>: Sync,
Option<StreamWorkerPool<S, F, I, M>>: Sync,
Mutex<Vec<(u64, BufferedRecord<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output, <M as MemoryPolicy>::Permit>)>>: Sync,
Option<fn(&<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output) -> usize>: Sync,
PhantomData<(M, N)>: Sync,
impl<S, C, F, I, M, N> Unpin for StreamDataLoader<S, C, F, I, M, N>where
Arc<S>: Unpin,
C: Unpin,
Arc<F>: Unpin,
Arc<I>: Unpin,
Option<StreamWorkerPool<S, F, I, M>>: Unpin,
Mutex<Vec<(u64, BufferedRecord<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output, <M as MemoryPolicy>::Permit>)>>: Unpin,
Option<fn(&<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output) -> usize>: Unpin,
PhantomData<(M, N)>: Unpin,
impl<S, C, F, I, M, N> UnsafeUnpin for StreamDataLoader<S, C, F, I, M, N>where
Arc<S>: UnsafeUnpin,
C: UnsafeUnpin,
Arc<F>: UnsafeUnpin,
Arc<I>: UnsafeUnpin,
Option<StreamWorkerPool<S, F, I, M>>: UnsafeUnpin,
Mutex<Vec<(u64, BufferedRecord<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output, <M as MemoryPolicy>::Permit>)>>: UnsafeUnpin,
Option<fn(&<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output) -> usize>: UnsafeUnpin,
PhantomData<(M, N)>: UnsafeUnpin,
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more