Skip to main content

StreamDataLoader

Struct StreamDataLoader 

Source
pub struct StreamDataLoader<S, C, F = IdentityTransformFactory, I = NoWorkerInit, M = MemoryDisabled, N = PinDisabled>{ /* private fields */ }
Expand description

Owned, re-iterable explicitly sharded stream loader.

Implementations§

Source§

impl<S, C, F, I, M, N> StreamDataLoader<S, C, F, I, M, N>

Source

pub fn len(&self) -> Option<usize>

Returns the exact global batch count when the factory is sized.

Source

pub fn is_empty(&self) -> Option<bool>

Returns whether the sized stream yields no batches.

Source

pub fn workers(&self) -> usize

Returns the configured worker count.

Source

pub fn effective_prefetch_factor(&self) -> NonZeroUsize

Returns the effective records-prefetched-per-worker factor.

Source

pub fn effective_prefetch_bytes(&self) -> Option<NonZeroUsize>

Returns the effective post-transform record byte budget.

Source

pub fn pin_memory_enabled(&self) -> bool

Returns whether recursive batch pinning was requested.

Source

pub fn pin_memory_status(&self) -> PinMemoryStatus

Returns the effective recursive pinning behavior.

Source

pub fn is_ordered(&self) -> bool

Returns whether ordered delivery is enabled.

Source

pub fn epoch(&self) -> u64

Returns the configured epoch.

Source

pub fn set_epoch(&mut self, epoch: u64)

Selects the epoch used by future iterator generations.

Source§

impl<S, C, F, I, M, N> StreamDataLoader<S, C, F, I, M, N>
where S: WorkerSourceFactory, F: TransformFactory<S::Sample> + Send + Sync + 'static, F::Transform: Send + 'static, <<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output: Send + 'static, <<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Error: Send + 'static, F::Error: Send + 'static, C: Collate<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output>, I: WorkerInit + Send + Sync + 'static, I::Error: Send + 'static, M: StreamMemoryPolicy + Send + 'static, N: StreamPinPolicy<S, C, F>,

Source

pub fn iter(&mut self) -> StreamLoaderIter<'_, S, C, F, I, M, N> ⓘ

Starts a fresh explicitly sharded worker generation.

Auto Trait Implementations§

§

impl<S, C, F = IdentityTransformFactory, I = NoWorkerInit, M = MemoryDisabled, N = PinDisabled> !Freeze for StreamDataLoader<S, C, F, I, M, N>

§

impl<S, C, F = IdentityTransformFactory, I = NoWorkerInit, M = MemoryDisabled, N = PinDisabled> !RefUnwindSafe for StreamDataLoader<S, C, F, I, M, N>

§

impl<S, C, F = IdentityTransformFactory, I = NoWorkerInit, M = MemoryDisabled, N = PinDisabled> !UnwindSafe for StreamDataLoader<S, C, F, I, M, N>

§

impl<S, C, F, I, M, N> Send for StreamDataLoader<S, C, F, I, M, N>
where Arc<S>: Send, C: Send, Arc<F>: Send, Arc<I>: Send, Option<StreamWorkerPool<S, F, I, M>>: Send, Mutex<Vec<(u64, BufferedRecord<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output, <M as MemoryPolicy>::Permit>)>>: Send, Option<fn(&<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output) -> usize>: Send, PhantomData<(M, N)>: Send,

§

impl<S, C, F, I, M, N> Sync for StreamDataLoader<S, C, F, I, M, N>
where Arc<S>: Sync, C: Sync, Arc<F>: Sync, Arc<I>: Sync, Option<StreamWorkerPool<S, F, I, M>>: Sync, Mutex<Vec<(u64, BufferedRecord<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output, <M as MemoryPolicy>::Permit>)>>: Sync, Option<fn(&<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output) -> usize>: Sync, PhantomData<(M, N)>: Sync,

§

impl<S, C, F, I, M, N> Unpin for StreamDataLoader<S, C, F, I, M, N>
where Arc<S>: Unpin, C: Unpin, Arc<F>: Unpin, Arc<I>: Unpin, Option<StreamWorkerPool<S, F, I, M>>: Unpin, Mutex<Vec<(u64, BufferedRecord<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output, <M as MemoryPolicy>::Permit>)>>: Unpin, Option<fn(&<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output) -> usize>: Unpin, PhantomData<(M, N)>: Unpin,

§

impl<S, C, F, I, M, N> UnsafeUnpin for StreamDataLoader<S, C, F, I, M, N>
where Arc<S>: UnsafeUnpin, C: UnsafeUnpin, Arc<F>: UnsafeUnpin, Arc<I>: UnsafeUnpin, Option<StreamWorkerPool<S, F, I, M>>: UnsafeUnpin, Mutex<Vec<(u64, BufferedRecord<<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output, <M as MemoryPolicy>::Permit>)>>: UnsafeUnpin, Option<fn(&<<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<<S as WorkerSourceFactory>::Sample>>::Output) -> usize>: UnsafeUnpin, PhantomData<(M, N)>: UnsafeUnpin,

Blanket Implementations§

Source§

impl<T> Any for T
where T: 'static + ?Sized,

Source§

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more
Source§

impl<T> Borrow<T> for T
where T: ?Sized,

Source§

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more
Source§

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source§

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more
Source§

impl<ST, DT> CastableFrom<ST, Initialized, Initialized> for DT
where ST: ?Sized, DT: ?Sized,

Source§

impl<ST, DT> CastableFrom<ST, Uninit, Uninit> for DT
where ST: ?Sized, DT: ?Sized,

Source§

impl<T> From<T> for T

Source§

fn from(t: T) -> T

Returns the argument unchanged.

Source§

impl<T, U> Into<U> for T
where U: From<T>,

Source§

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source§

impl<T> Read<Exclusive, BecauseExclusive> for T
where T: ?Sized,

Source§

impl<T> Same for T

Source§

type Output = T

Should always be Self
Source§

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source§

type Error = !

The type returned in the event of a conversion error.
Source§

fn try_from(value: U) -> Result<T, !>

Performs the conversion.
Source§

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source§

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.
Source§

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.
Source§

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

Source§

fn vzip(self) -> V