Skip to main content

rusttorch_data/
stream.rs

1//! Load lazy, independently owned stream shards with worker threads.
2//!
3//! [`StreamDataLoaderBuilder`] has a complete sharding example and explains
4//! ordering, batching, prefetch and cleanup. [`ExactStreamDataLoader`] adds
5//! resumable cursors for replay-safe sources. Use [`crate::batches`] for an
6//! ordinary iterator that needs no worker ownership or checkpointing.
7
8// The ordered window is a loader-owned flat slot vector reserved and aggregate-
9// checked before source callbacks, then reused across generations. Lookup is
10// linear in this deliberately bounded window; wider source disorder increases
11// both its memory and scan cost. Byte-bounded streams additionally use one
12// front-waiter slot per shard to distinguish a slow next record from a missing ID.
13
14use std::{
15    convert::Infallible,
16    marker::PhantomData,
17    mem::{MaybeUninit, size_of},
18    num::NonZeroUsize,
19    panic::{AssertUnwindSafe, catch_unwind},
20    sync::{Arc, Mutex, atomic::AtomicUsize},
21    thread::{self, JoinHandle},
22    time::Duration,
23};
24
25use crossbeam_channel::{Receiver, RecvError, Sender, TryRecvError, after, bounded, select};
26
27use rusttorch_core::{Device, Result, RustTorchError, available_devices};
28
29use crate::memory::{BudgetError, ByteBudget, MemoryDisabled, MemoryEnabled, MemoryPolicy};
30use crate::worker::WorkerRunContext;
31#[path = "stream_checkpoint.rs"]
32mod exact;
33use crate::{
34    Auto, CancellationToken, CloneTransformFactory, Collate, Deadline, DefaultCollator, Explicit,
35    IdentityTransformFactory, LoaderError, MemoryFootprint, NoWorkerInit, PinDisabled, PinEnabled,
36    PinMemory, PinMemoryStatus, PipelineError, TaskContext, Transform, TransformFactory,
37    WorkerContext, WorkerInfo, WorkerInit, with_worker_info,
38};
39pub use exact::{ExactStreamDataLoader, ExactStreamLoaderIter};
40
41/// A record's position in one ordered global stream generation.
42///
43/// Assign positions `0, 1, 2, ...` across all shards together. The loader uses
44/// them to restore global order and rejects duplicate or missing positions.
45/// Keep sample identity separate in [`LogicalSampleId`] when the same sample
46/// can appear at a different position in another epoch.
47#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
48pub struct SequenceId(u64);
49
50impl SequenceId {
51    /// Wraps a global position; ordering and completeness are checked while loading.
52    pub const fn new(value: u64) -> Self {
53        Self(value)
54    }
55
56    /// Returns the integer representation.
57    pub const fn get(self) -> u64 {
58        self.0
59    }
60
61    /// Returns the next identifier, or `None` at the integer boundary.
62    pub const fn checked_next(self) -> Option<Self> {
63        match self.0.checked_add(1) {
64            Some(value) => Some(Self(value)),
65            None => None,
66        }
67    }
68}
69
70/// Stable sample identity used by task randomness and stream checkpoints.
71///
72/// Unlike [`SequenceId`], this identifies the underlying sample rather than its
73/// delivery position. Keep it stable for the same sample when worker scheduling
74/// changes. [`StreamDataLoaderBuilder`] shows a source where both IDs coincide.
75#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
76pub struct LogicalSampleId(u64);
77
78impl LogicalSampleId {
79    /// Wraps a caller-assigned stable sample identity.
80    pub const fn new(value: u64) -> Self {
81        Self(value)
82    }
83
84    /// Returns the integer representation.
85    pub const fn get(self) -> u64 {
86        self.0
87    }
88
89    /// Returns the next identifier, or `None` at the integer boundary.
90    pub const fn checked_next(self) -> Option<Self> {
91        match self.0.checked_add(1) {
92            Some(value) => Some(Self(value)),
93            None => None,
94        }
95    }
96}
97
98/// An owned sample plus its ordering and reproducibility identifiers.
99///
100/// Ordered loaders require a `sequence`; completion-order loaders may use
101/// `None`. All records require a stable `logical_id` for sample-local randomness.
102/// See [`StreamDataLoaderBuilder`] for a complete lazy source implementation.
103#[derive(Clone, Debug, Eq, PartialEq)]
104pub struct WorkerRecord<T> {
105    /// Position in the global ordered generation, when supplied.
106    pub sequence: Option<SequenceId>,
107    /// Stable task-randomness and checkpoint identity.
108    pub logical_id: LogicalSampleId,
109    /// Owned source sample.
110    pub sample: T,
111}
112
113/// Open an independent input shard for each worker and each iteration.
114///
115/// Implement this for sharded files, partitioned database scans or other sources
116/// that can be opened separately. Use `context.info.id` and
117/// `context.info.num_workers` to select disjoint records; the loader does not
118/// partition an arbitrary source automatically. Returning the same full stream
119/// from every worker duplicates its records.
120///
121/// [`StreamDataLoaderBuilder`] provides a complete lazy modulo-shard example.
122/// Retain `WorkerContext` when a decoder needs cancellation/deadline information.
123/// Report the total record count across all shards through [`Self::exact_len`]
124/// only when it is exact; return `None` for an unknown-length source.
125pub trait WorkerSourceFactory: Send + Sync + 'static {
126    /// Sample produced by every shard.
127    type Sample: Send + 'static;
128    /// Typed source creation or iteration failure.
129    type Error: Send + 'static;
130    /// Independently owned shard iterator.
131    type Source: Iterator<Item = std::result::Result<WorkerRecord<Self::Sample>, Self::Error>>
132        + Send
133        + 'static;
134
135    /// Creates this worker's shard for the supplied generation context.
136    fn create(&self, worker: WorkerContext) -> std::result::Result<Self::Source, Self::Error>;
137
138    /// Returns the exact global record count when known.
139    fn exact_len(&self) -> Option<usize> {
140        None
141    }
142}
143
144type TransformOutput<S, F> =
145    <<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<
146        <S as WorkerSourceFactory>::Sample,
147    >>::Output;
148type TransformFailure<S, F> =
149    <<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<
150        <S as WorkerSourceFactory>::Sample,
151    >>::Error;
152type StreamPipelineError<S, C, F, I> = PipelineError<
153    <S as WorkerSourceFactory>::Error,
154    TransformFailure<S, F>,
155    <C as Collate<TransformOutput<S, F>>>::Error,
156    <F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Error,
157    <I as WorkerInit>::Error,
158>;
159type StreamLoaderError<S, C, F, I> = LoaderError<StreamPipelineError<S, C, F, I>>;
160type ReassemblyEntry<T, P> = (u64, BufferedRecord<T, P>);
161type RetainedReassembly<T, P> = Mutex<Vec<ReassemblyEntry<T, P>>>;
162type StreamFootprint<S, F> = fn(&TransformOutput<S, F>) -> usize;
163
164enum StreamByteFailure {
165    MemoryLimit {
166        limit: usize,
167        actual: usize,
168    },
169    Protocol {
170        sequence: Option<u64>,
171        reason: String,
172    },
173}
174
175trait StreamMemoryPolicy: MemoryPolicy {
176    type Waiters;
177    type Waiting: Send + 'static;
178    type Failure: Send + 'static;
179
180    fn waiter_bytes(workers: usize) -> Result<usize>;
181    fn preflight_waiters(workers: usize) -> Result<()>;
182    fn new_waiters(workers: usize) -> Result<Self::Waiters>;
183    fn waiters(waiters: &Self::Waiters) -> Option<&[Option<u64>]>;
184    fn waiters_mut(waiters: &mut Self::Waiters) -> Option<&mut [Option<u64>]>;
185    fn waiting(sequence: u64) -> Option<Self::Waiting>;
186    fn waiting_sequence(waiting: Self::Waiting) -> u64;
187    fn memory_limit(limit: usize, actual: usize) -> Option<Self::Failure>;
188    fn protocol(sequence: Option<u64>, reason: String) -> Option<Self::Failure>;
189    fn map_failure<E>(
190        failure: Self::Failure,
191        worker: usize,
192        sequence: Option<u64>,
193        logical_id: Option<u64>,
194    ) -> LoaderError<E>;
195}
196
197impl StreamMemoryPolicy for MemoryDisabled {
198    type Waiters = ();
199    type Waiting = Infallible;
200    type Failure = Infallible;
201
202    fn waiter_bytes(_workers: usize) -> Result<usize> {
203        Ok(0)
204    }
205
206    fn preflight_waiters(_workers: usize) -> Result<()> {
207        Ok(())
208    }
209
210    fn new_waiters(_workers: usize) -> Result<Self::Waiters> {
211        Ok(())
212    }
213
214    fn waiters(_waiters: &Self::Waiters) -> Option<&[Option<u64>]> {
215        None
216    }
217
218    fn waiters_mut(_waiters: &mut Self::Waiters) -> Option<&mut [Option<u64>]> {
219        None
220    }
221
222    fn waiting(_sequence: u64) -> Option<Self::Waiting> {
223        None
224    }
225
226    fn waiting_sequence(waiting: Self::Waiting) -> u64 {
227        match waiting {}
228    }
229
230    fn memory_limit(_limit: usize, _actual: usize) -> Option<Self::Failure> {
231        None
232    }
233
234    fn protocol(_sequence: Option<u64>, _reason: String) -> Option<Self::Failure> {
235        None
236    }
237
238    fn map_failure<E>(
239        failure: Self::Failure,
240        _worker: usize,
241        _sequence: Option<u64>,
242        _logical_id: Option<u64>,
243    ) -> LoaderError<E> {
244        match failure {}
245    }
246}
247
248impl StreamMemoryPolicy for MemoryEnabled {
249    type Waiters = Vec<Option<u64>>;
250    type Waiting = u64;
251    type Failure = StreamByteFailure;
252
253    fn waiter_bytes(workers: usize) -> Result<usize> {
254        workers
255            .checked_mul(size_of::<Option<u64>>())
256            .ok_or_else(|| capacity_error("stream byte waiter storage exceeds usize"))
257    }
258
259    fn preflight_waiters(workers: usize) -> Result<()> {
260        preflight_vec::<Option<u64>>(workers, "stream byte waiter state")
261    }
262
263    fn new_waiters(workers: usize) -> Result<Self::Waiters> {
264        let mut waiters = Vec::new();
265        reserve_exact(&mut waiters, workers, "stream byte waiter state")?;
266        waiters.resize(workers, None);
267        Ok(waiters)
268    }
269
270    fn waiters(waiters: &Self::Waiters) -> Option<&[Option<u64>]> {
271        Some(waiters)
272    }
273
274    fn waiters_mut(waiters: &mut Self::Waiters) -> Option<&mut [Option<u64>]> {
275        Some(waiters)
276    }
277
278    fn waiting(sequence: u64) -> Option<Self::Waiting> {
279        Some(sequence)
280    }
281
282    fn waiting_sequence(waiting: Self::Waiting) -> u64 {
283        waiting
284    }
285
286    fn memory_limit(limit: usize, actual: usize) -> Option<Self::Failure> {
287        Some(StreamByteFailure::MemoryLimit { limit, actual })
288    }
289
290    fn protocol(sequence: Option<u64>, reason: String) -> Option<Self::Failure> {
291        Some(StreamByteFailure::Protocol { sequence, reason })
292    }
293
294    fn map_failure<E>(
295        failure: Self::Failure,
296        worker: usize,
297        sequence: Option<u64>,
298        logical_id: Option<u64>,
299    ) -> LoaderError<E> {
300        match failure {
301            StreamByteFailure::MemoryLimit { limit, actual } => LoaderError::MemoryLimit {
302                batch: None,
303                worker: Some(worker),
304                sequence,
305                logical_id,
306                limit,
307                actual,
308            },
309            StreamByteFailure::Protocol { sequence, reason } => {
310                LoaderError::StreamProtocol { sequence, reason }
311            }
312        }
313    }
314}
315
316#[doc(hidden)]
317pub trait StreamPinPolicy<S, C, F>
318where
319    S: WorkerSourceFactory,
320    F: TransformFactory<S::Sample>,
321    C: Collate<TransformOutput<S, F>>,
322{
323    fn pin(batch: C::Batch, device: Device) -> Result<C::Batch>;
324}
325
326impl<S, C, F> StreamPinPolicy<S, C, F> for PinDisabled
327where
328    S: WorkerSourceFactory,
329    F: TransformFactory<S::Sample>,
330    C: Collate<TransformOutput<S, F>>,
331{
332    fn pin(batch: C::Batch, _device: Device) -> Result<C::Batch> {
333        Ok(batch)
334    }
335}
336
337impl<S, C, F, Q> StreamPinPolicy<S, C, F> for PinEnabled<Q>
338where
339    S: WorkerSourceFactory,
340    F: TransformFactory<S::Sample>,
341    C: Collate<TransformOutput<S, F>>,
342    C::Batch: PinMemory,
343{
344    fn pin(batch: C::Batch, device: Device) -> Result<C::Batch> {
345        batch.pin_memory(device)
346    }
347}
348
349type StreamCompletionFor<S, F, I, M> = StreamCompletion<
350    TransformOutput<S, F>,
351    <M as MemoryPolicy>::Permit,
352    <M as StreamMemoryPolicy>::Waiting,
353    <M as StreamMemoryPolicy>::Failure,
354    <S as WorkerSourceFactory>::Error,
355    TransformFailure<S, F>,
356    <F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Error,
357    <I as WorkerInit>::Error,
358>;
359type StreamFailureFor<S, F, I, M> = StreamFailure<
360    <S as WorkerSourceFactory>::Error,
361    TransformFailure<S, F>,
362    <F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Error,
363    <I as WorkerInit>::Error,
364    <M as StreamMemoryPolicy>::Failure,
365>;
366type StreamResult<T, P, W, ME, SE, TE, FE, IE> =
367    std::result::Result<StreamMessage<T, P, W>, StreamFailure<SE, TE, FE, IE, ME>>;
368type StreamCompletionSender<T, P, W, ME, SE, TE, FE, IE> =
369    Sender<StreamCompletion<T, P, W, ME, SE, TE, FE, IE>>;
370
371enum StreamFailure<SE, TE, FE, IE, ME> {
372    Source(SE),
373    Transform(TE),
374    TransformInit(FE),
375    WorkerInit(IE),
376    Memory(ME),
377    Panic,
378}
379
380enum StreamMessage<T, P, W> {
381    Record(WorkerRecord<T>, P),
382    Waiting(W),
383    End,
384}
385
386struct StreamCompletion<T, P, W, ME, SE, TE, FE, IE> {
387    worker: usize,
388    generation: u64,
389    sequence: Option<u64>,
390    logical_id: Option<u64>,
391    holds_credit: bool,
392    result: StreamResult<T, P, W, ME, SE, TE, FE, IE>,
393}
394
395enum StreamControl {
396    Begin(WorkerRunContext),
397}
398
399enum StreamReceive<S, F, I, M>
400where
401    S: WorkerSourceFactory,
402    F: TransformFactory<S::Sample>,
403    I: WorkerInit,
404    M: StreamMemoryPolicy,
405{
406    Completion(StreamCompletionFor<S, F, I, M>),
407    Timeout,
408    Cancelled,
409    Closed,
410}
411
412enum ReadyFirst<T> {
413    Completion(T),
414    Timeout,
415    Cancelled,
416    Closed,
417}
418
419const MAX_STREAM_QUEUE_ALLOCATION_BYTES: usize = 64 * 1024 * 1024;
420const CHANNEL_CONTROL_BLOCK_ALLOWANCE_BYTES: usize = 2 * 1024;
421
422#[allow(dead_code)]
423#[repr(C)]
424struct CapacitySlot<T> {
425    stamp: AtomicUsize,
426    message: MaybeUninit<T>,
427}
428
429fn validate_stream_capacity<S, F, I, M>(
430    workers: usize,
431    prefetch_factor: usize,
432    outstanding: usize,
433    batch_size: usize,
434    reassembly_slots: usize,
435) -> Result<()>
436where
437    S: WorkerSourceFactory,
438    F: TransformFactory<S::Sample>,
439    I: WorkerInit,
440    M: StreamMemoryPolicy,
441{
442    validate_stream_capacity_extra::<S, F, I, M>(
443        workers,
444        prefetch_factor,
445        outstanding,
446        batch_size,
447        reassembly_slots,
448        0,
449    )
450}
451
452fn validate_stream_capacity_extra<S, F, I, M>(
453    workers: usize,
454    prefetch_factor: usize,
455    outstanding: usize,
456    batch_size: usize,
457    reassembly_slots: usize,
458    extra: usize,
459) -> Result<()>
460where
461    S: WorkerSourceFactory,
462    F: TransformFactory<S::Sample>,
463    I: WorkerInit,
464    M: StreamMemoryPolicy,
465{
466    let expected = workers
467        .checked_mul(prefetch_factor)
468        .ok_or_else(|| capacity_error("workers multiplied by prefetch_factor exceeds usize"))?;
469    if expected != outstanding {
470        return Err(capacity_error(
471            "outstanding capacity does not match workers multiplied by prefetch_factor",
472        ));
473    }
474    for capacity in [1, prefetch_factor, outstanding] {
475        capacity
476            .checked_add(1)
477            .and_then(usize::checked_next_power_of_two)
478            .and_then(|mark_bit| mark_bit.checked_mul(2))
479            .ok_or_else(|| capacity_error("bounded channel ring arithmetic overflowed"))?;
480    }
481
482    let credit_bytes = outstanding
483        .checked_mul(size_of::<CapacitySlot<()>>())
484        .ok_or_else(|| capacity_error("stream credit storage exceeds usize"))?;
485    let result_bytes = outstanding
486        .checked_mul(size_of::<CapacitySlot<StreamCompletionFor<S, F, I, M>>>())
487        .ok_or_else(|| capacity_error("stream result storage exceeds usize"))?;
488    let control_bytes = workers
489        .checked_mul(size_of::<CapacitySlot<StreamControl>>())
490        .ok_or_else(|| capacity_error("stream control storage exceeds usize"))?;
491    let batch_bytes = batch_size
492        .checked_mul(size_of::<TransformOutput<S, F>>())
493        .and_then(|bytes| bytes.checked_mul(2))
494        .ok_or_else(|| capacity_error("stream batch storage exceeds usize"))?;
495    let reassembly_bytes = reassembly_slots
496        .checked_mul(size_of::<
497            ReassemblyEntry<TransformOutput<S, F>, <M as MemoryPolicy>::Permit>,
498        >())
499        .ok_or_else(|| capacity_error("ordered stream reassembly exceeds usize"))?;
500    let per_worker = size_of::<WorkerInfo>()
501        .checked_add(size_of::<(Sender<()>, Receiver<()>)>())
502        .and_then(|bytes| {
503            bytes.checked_add(size_of::<(Sender<StreamControl>, Receiver<StreamControl>)>())
504        })
505        .and_then(|bytes| bytes.checked_add(size_of::<Sender<()>>()))
506        .and_then(|bytes| bytes.checked_add(size_of::<Sender<StreamControl>>()))
507        .and_then(|bytes| bytes.checked_add(size_of::<JoinHandle<()>>()))
508        .and_then(|bytes| bytes.checked_add(size_of::<bool>()))
509        .ok_or_else(|| capacity_error("stream worker bookkeeping exceeds usize"))?;
510    let waiter_bytes = M::waiter_bytes(workers)?;
511    let bookkeeping = workers
512        .checked_mul(per_worker)
513        .and_then(|bytes| bytes.checked_add(waiter_bytes))
514        .ok_or_else(|| capacity_error("stream worker bookkeeping exceeds usize"))?;
515    let channel_count = workers
516        .checked_mul(2)
517        .and_then(|count| count.checked_add(1))
518        .ok_or_else(|| capacity_error("stream channel count exceeds usize"))?;
519    let channel_bytes = channel_count
520        .checked_mul(CHANNEL_CONTROL_BLOCK_ALLOWANCE_BYTES)
521        .ok_or_else(|| capacity_error("stream channel controls exceed usize"))?;
522    let aggregate = credit_bytes
523        .checked_add(result_bytes)
524        .and_then(|bytes| bytes.checked_add(control_bytes))
525        .and_then(|bytes| bytes.checked_add(batch_bytes))
526        .and_then(|bytes| bytes.checked_add(reassembly_bytes))
527        .and_then(|bytes| bytes.checked_add(bookkeeping))
528        .and_then(|bytes| bytes.checked_add(channel_bytes))
529        .and_then(|bytes| bytes.checked_add(extra))
530        .ok_or_else(|| capacity_error("aggregate stream queue storage exceeds usize"))?;
531    if aggregate > MAX_STREAM_QUEUE_ALLOCATION_BYTES {
532        return Err(capacity_error(format!(
533            "aggregate stream queue storage requires {aggregate} bytes, above the {MAX_STREAM_QUEUE_ALLOCATION_BYTES}-byte safety ceiling"
534        )));
535    }
536    preflight_channel_storage::<()>(outstanding)?;
537    preflight_channel_storage::<StreamControl>(workers)?;
538    preflight_channel_storage::<StreamCompletionFor<S, F, I, M>>(outstanding)?;
539    preflight_vec::<TransformOutput<S, F>>(batch_size, "stream batch")?;
540    M::preflight_waiters(workers)?;
541    Ok(())
542}
543
544fn preflight_channel_storage<T>(capacity: usize) -> Result<()> {
545    let mut reservation: Vec<MaybeUninit<CapacitySlot<T>>> = Vec::new();
546    reservation
547        .try_reserve_exact(capacity)
548        .map_err(|error| capacity_error(format!("bounded stream storage is unavailable: {error}")))
549}
550
551fn preflight_vec<T>(capacity: usize, label: &'static str) -> Result<()> {
552    let mut reservation: Vec<T> = Vec::new();
553    reservation
554        .try_reserve_exact(capacity)
555        .map_err(|error| capacity_error(format!("{label} capacity is unavailable: {error}")))
556}
557
558fn capacity_error(reason: impl Into<String>) -> RustTorchError {
559    invalid_configuration("prefetch_factor", reason)
560}
561
562struct StreamWorkerPool<S, F, I, M>
563where
564    S: WorkerSourceFactory,
565    F: TransformFactory<S::Sample>,
566    I: WorkerInit,
567    M: StreamMemoryPolicy,
568{
569    controls: Vec<Sender<StreamControl>>,
570    credits: Vec<Sender<()>>,
571    results: Option<Receiver<StreamCompletionFor<S, F, I, M>>>,
572    handles: Vec<JoinHandle<()>>,
573    shutdown: CancellationToken,
574    active: Option<WorkerRunContext>,
575    terminal: Vec<bool>,
576    terminal_count: usize,
577    fatal_exit: bool,
578    poisoned: bool,
579}
580
581impl<S, F, I, M> StreamWorkerPool<S, F, I, M>
582where
583    S: WorkerSourceFactory,
584    F: TransformFactory<S::Sample> + Send + Sync + 'static,
585    F::Transform: Send + 'static,
586    TransformOutput<S, F>: Send + 'static,
587    TransformFailure<S, F>: Send + 'static,
588    F::Error: Send + 'static,
589    I: WorkerInit + Send + Sync + 'static,
590    I::Error: Send + 'static,
591    M: StreamMemoryPolicy + Send + 'static,
592{
593    #[allow(clippy::too_many_arguments)]
594    fn new(
595        source_factory: Arc<S>,
596        transform_factory: Arc<F>,
597        initializer: Arc<I>,
598        configuration: StreamConfiguration,
599        generation: u64,
600        outstanding: usize,
601        reassembly_slots: usize,
602        footprint: Option<StreamFootprint<S, F>>,
603    ) -> Result<Self> {
604        validate_stream_capacity::<S, F, I, M>(
605            configuration.workers,
606            configuration.prefetch_factor,
607            outstanding,
608            configuration.batch_size,
609            reassembly_slots,
610        )?;
611
612        let (result_sender, result_receiver) =
613            bounded_checked(outstanding, "stream result channel")?;
614        let mut credit_lanes = Vec::new();
615        let mut control_lanes = Vec::new();
616        reserve_exact(
617            &mut credit_lanes,
618            configuration.workers,
619            "stream credit channels",
620        )?;
621        reserve_exact(
622            &mut control_lanes,
623            configuration.workers,
624            "stream control channels",
625        )?;
626        for _ in 0..configuration.workers {
627            let (credit_sender, credit_receiver) =
628                bounded_checked(configuration.prefetch_factor, "stream credit channel")?;
629            for _ in 0..configuration.prefetch_factor {
630                credit_sender.try_send(()).map_err(|_| {
631                    capacity_error("validated stream credit channel rejected a token")
632                })?;
633            }
634            credit_lanes.push((credit_sender, credit_receiver));
635            control_lanes.push(bounded_checked(1, "stream control channel")?);
636        }
637
638        let shutdown = CancellationToken::new();
639        let mut pool = Self {
640            controls: Vec::new(),
641            credits: Vec::new(),
642            results: Some(result_receiver),
643            handles: Vec::new(),
644            shutdown: shutdown.clone(),
645            active: None,
646            terminal: Vec::new(),
647            terminal_count: 0,
648            fatal_exit: false,
649            poisoned: false,
650        };
651        reserve_exact(&mut pool.controls, configuration.workers, "stream controls")?;
652        reserve_exact(&mut pool.credits, configuration.workers, "stream credits")?;
653        reserve_exact(&mut pool.handles, configuration.workers, "stream workers")?;
654        reserve_exact(
655            &mut pool.terminal,
656            configuration.workers,
657            "stream terminal state",
658        )?;
659        pool.terminal.resize(configuration.workers, false);
660
661        for (id, ((credit_sender, credit_receiver), (control_sender, control_receiver))) in
662            credit_lanes.into_iter().zip(control_lanes).enumerate()
663        {
664            let info = WorkerInfo::from_loader_seed(
665                id,
666                configuration.workers,
667                configuration.loader_seed,
668                configuration.rank,
669                generation,
670            )?;
671            pool.credits.push(credit_sender);
672            pool.controls.push(control_sender);
673            let source_factory = Arc::clone(&source_factory);
674            let transform_factory = Arc::clone(&transform_factory);
675            let initializer = Arc::clone(&initializer);
676            let results = result_sender.clone();
677            let shutdown = shutdown.clone();
678            let handle = thread::Builder::new()
679                .name(format!("rusttorch-stream-worker-{id}"))
680                .spawn(move || {
681                    run_stream_worker::<S, F, I, M>(
682                        info,
683                        generation,
684                        source_factory,
685                        transform_factory,
686                        initializer,
687                        control_receiver,
688                        credit_receiver,
689                        results,
690                        shutdown,
691                        footprint,
692                        configuration.ordered,
693                    );
694                })
695                .map_err(|error| RustTorchError::BackendUnavailable {
696                    backend: "data loader stream worker threads",
697                    reason: error.to_string(),
698                })?;
699            pool.handles.push(handle);
700        }
701        drop(result_sender);
702        Ok(pool)
703    }
704}
705
706impl<S, F, I, M> StreamWorkerPool<S, F, I, M>
707where
708    S: WorkerSourceFactory,
709    F: TransformFactory<S::Sample>,
710    I: WorkerInit,
711    M: StreamMemoryPolicy,
712{
713    fn start_generation(&mut self, context: WorkerRunContext) -> std::result::Result<(), ()> {
714        if self.poisoned || self.active.is_some() || self.shutdown.is_cancelled() {
715            return Err(());
716        }
717        self.terminal.fill(false);
718        self.terminal_count = 0;
719        self.fatal_exit = false;
720        self.active = Some(context.clone());
721        for control in &self.controls {
722            if control.send(StreamControl::Begin(context.clone())).is_err() {
723                self.poisoned = true;
724                self.shutdown();
725                return Err(());
726            }
727        }
728        Ok(())
729    }
730
731    fn receive(
732        &self,
733        context: &WorkerRunContext,
734        timeout: Option<Duration>,
735    ) -> StreamReceive<S, F, I, M> {
736        let results = self
737            .results
738            .as_ref()
739            .expect("stream result receiver exists before shutdown");
740        match receive_ready_first(results, context, &self.shutdown, timeout) {
741            ReadyFirst::Completion(completion) => StreamReceive::Completion(completion),
742            ReadyFirst::Timeout => StreamReceive::Timeout,
743            ReadyFirst::Cancelled => StreamReceive::Cancelled,
744            ReadyFirst::Closed => StreamReceive::Closed,
745        }
746    }
747
748    fn account(
749        &mut self,
750        completion: &StreamCompletionFor<S, F, I, M>,
751    ) -> std::result::Result<(), ()> {
752        let fatal = matches!(
753            &completion.result,
754            Err(StreamFailure::TransformInit(_)
755                | StreamFailure::WorkerInit(_)
756                | StreamFailure::Panic)
757        );
758        let terminal = fatal || matches!(&completion.result, Ok(StreamMessage::End));
759        self.account_parts(completion.worker, completion.holds_credit, terminal, fatal)
760    }
761
762    fn account_parts(
763        &mut self,
764        worker: usize,
765        holds_credit: bool,
766        terminal: bool,
767        fatal: bool,
768    ) -> std::result::Result<(), ()> {
769        if holds_credit {
770            self.return_credit(worker)?;
771        }
772        if fatal {
773            self.fatal_exit = true;
774        }
775        if terminal {
776            let Some(seen) = self.terminal.get_mut(worker) else {
777                return Err(());
778            };
779            if !*seen {
780                *seen = true;
781                self.terminal_count += 1;
782            }
783        }
784        Ok(())
785    }
786
787    fn return_credit(&self, worker: usize) -> std::result::Result<(), ()> {
788        let Some(credit) = self.credits.get(worker) else {
789            return Err(());
790        };
791        credit.try_send(()).map_err(|_| ())
792    }
793
794    fn all_terminal(&self) -> bool {
795        self.terminal_count == self.terminal.len()
796    }
797
798    fn quiesce(&mut self, generation: u64) -> std::result::Result<(), ()> {
799        let Some(active) = self.active.take() else {
800            return Ok(());
801        };
802        active.cancel();
803        while !self.all_terminal() {
804            let completion = self.results.as_ref().ok_or(())?.recv().map_err(|_| ())?;
805            if completion.generation == generation {
806                self.account(&completion)?;
807            } else if completion.holds_credit {
808                self.return_credit(completion.worker)?;
809            }
810        }
811        while let Some(results) = self.results.as_ref() {
812            match results.try_recv() {
813                Ok(completion) => self.account(&completion)?,
814                Err(TryRecvError::Empty) => break,
815                Err(TryRecvError::Disconnected) => return Err(()),
816            }
817        }
818        if self.fatal_exit {
819            self.poisoned = true;
820            self.shutdown();
821            return Err(());
822        }
823        Ok(())
824    }
825
826    fn poison(&mut self) {
827        self.poisoned = true;
828        self.shutdown();
829    }
830
831    fn shutdown(&mut self) {
832        if let Some(active) = self.active.take() {
833            active.cancel();
834        }
835        self.shutdown.cancel();
836        self.controls.clear();
837        self.credits.clear();
838        self.results.take();
839        for handle in self.handles.drain(..) {
840            let _ = handle.join();
841        }
842    }
843}
844
845impl<S, F, I, M> Drop for StreamWorkerPool<S, F, I, M>
846where
847    S: WorkerSourceFactory,
848    F: TransformFactory<S::Sample>,
849    I: WorkerInit,
850    M: StreamMemoryPolicy,
851{
852    fn drop(&mut self) {
853        self.shutdown();
854    }
855}
856
857fn receive_ready_first<T>(
858    results: &Receiver<T>,
859    context: &WorkerRunContext,
860    shutdown: &CancellationToken,
861    timeout: Option<Duration>,
862) -> ReadyFirst<T> {
863    match results.try_recv() {
864        Ok(completion) => return ReadyFirst::Completion(completion),
865        Err(TryRecvError::Disconnected) => return ReadyFirst::Closed,
866        Err(TryRecvError::Empty) => {}
867    }
868    if let Some(timeout) = timeout {
869        let timer = after(timeout);
870        select! {
871            recv(results) -> result => map_ready_receive(result),
872            recv(context.cancellation.signal()) -> _ => match results.try_recv() {
873                Ok(completion) => ReadyFirst::Completion(completion),
874                Err(TryRecvError::Empty) => ReadyFirst::Cancelled,
875                Err(TryRecvError::Disconnected) => ReadyFirst::Closed,
876            },
877            recv(shutdown.signal()) -> _ => ReadyFirst::Closed,
878            recv(timer) -> _ => match results.try_recv() {
879                Ok(completion) => ReadyFirst::Completion(completion),
880                Err(TryRecvError::Empty) => ReadyFirst::Timeout,
881                Err(TryRecvError::Disconnected) => ReadyFirst::Closed,
882            },
883        }
884    } else {
885        select! {
886            recv(results) -> result => map_ready_receive(result),
887            recv(context.cancellation.signal()) -> _ => match results.try_recv() {
888                Ok(completion) => ReadyFirst::Completion(completion),
889                Err(TryRecvError::Empty) => ReadyFirst::Cancelled,
890                Err(TryRecvError::Disconnected) => ReadyFirst::Closed,
891            },
892            recv(shutdown.signal()) -> _ => ReadyFirst::Closed,
893        }
894    }
895}
896
897fn map_ready_receive<T>(result: std::result::Result<T, RecvError>) -> ReadyFirst<T> {
898    match result {
899        Ok(completion) => ReadyFirst::Completion(completion),
900        Err(_) => ReadyFirst::Closed,
901    }
902}
903
904fn reserve_exact<T>(values: &mut Vec<T>, capacity: usize, label: &'static str) -> Result<()> {
905    values
906        .try_reserve_exact(capacity)
907        .map_err(|error| capacity_error(format!("{label} capacity is unavailable: {error}")))
908}
909
910fn bounded_checked<T>(capacity: usize, label: &'static str) -> Result<(Sender<T>, Receiver<T>)> {
911    catch_unwind(AssertUnwindSafe(|| bounded(capacity))).map_err(|_| {
912        capacity_error(format!(
913            "{label} rejected the validated capacity {capacity}"
914        ))
915    })
916}
917
918#[allow(clippy::too_many_arguments)]
919fn run_stream_worker<S, F, I, M>(
920    info: WorkerInfo,
921    initialization_generation: u64,
922    source_factory: Arc<S>,
923    transform_factory: Arc<F>,
924    initializer: Arc<I>,
925    controls: Receiver<StreamControl>,
926    credits: Receiver<()>,
927    results: Sender<StreamCompletionFor<S, F, I, M>>,
928    shutdown: CancellationToken,
929    footprint: Option<StreamFootprint<S, F>>,
930    ordered: bool,
931) where
932    S: WorkerSourceFactory,
933    F: TransformFactory<S::Sample> + Send + Sync + 'static,
934    F::Transform: Send + 'static,
935    TransformOutput<S, F>: Send + 'static,
936    TransformFailure<S, F>: Send + 'static,
937    F::Error: Send + 'static,
938    I: WorkerInit + Send + Sync + 'static,
939    I::Error: Send + 'static,
940    M: StreamMemoryPolicy + Send + 'static,
941{
942    let lifecycle = WorkerContext::new(info, shutdown.clone(), shutdown.paired_deadline());
943    let mut active: Option<(WorkerRunContext, Option<u64>, Option<u64>, bool)> = None;
944    let outcome = catch_unwind(AssertUnwindSafe(|| {
945        with_worker_info(info, || {
946            if let Err(error) = initializer.initialize(&lifecycle) {
947                send_stream_failure(
948                    &results,
949                    &shutdown,
950                    info.id,
951                    initialization_generation,
952                    None,
953                    None,
954                    false,
955                    StreamFailure::WorkerInit(error),
956                );
957                return true;
958            }
959            let mut transform = match transform_factory.create(Some(&lifecycle)) {
960                Ok(transform) => transform,
961                Err(error) => {
962                    send_stream_failure(
963                        &results,
964                        &shutdown,
965                        info.id,
966                        initialization_generation,
967                        None,
968                        None,
969                        false,
970                        StreamFailure::TransformInit(error),
971                    );
972                    return true;
973                }
974            };
975
976            loop {
977                let run = select! {
978                    recv(controls) -> control => match control {
979                        Ok(StreamControl::Begin(run)) => run,
980                        Err(_) => return false,
981                    },
982                    recv(shutdown.signal()) -> _ => return false,
983                };
984                active = Some((run.clone(), None, None, false));
985                run_stream_generation::<S, F, I, M>(
986                    info,
987                    &source_factory,
988                    &mut transform,
989                    &credits,
990                    &results,
991                    &shutdown,
992                    &run,
993                    &mut active,
994                    footprint,
995                    ordered,
996                );
997                active = None;
998                if shutdown.is_cancelled() {
999                    return false;
1000                }
1001            }
1002        })
1003    }));
1004    let fatal = match outcome {
1005        Ok(fatal) => fatal,
1006        Err(_) => {
1007            let (generation, sequence, logical_id, holds_credit) = active
1008                .as_ref()
1009                .map(|(run, sequence, logical_id, held)| {
1010                    (run.generation, *sequence, *logical_id, *held)
1011                })
1012                .unwrap_or((initialization_generation, None, None, false));
1013            send_stream_failure(
1014                &results,
1015                &shutdown,
1016                info.id,
1017                generation,
1018                sequence,
1019                logical_id,
1020                holds_credit,
1021                StreamFailure::Panic,
1022            );
1023            true
1024        }
1025    };
1026    if fatal {
1027        shutdown.wait_cancelled();
1028    }
1029}
1030
1031#[allow(clippy::too_many_arguments)]
1032fn run_stream_generation<S, F, I, M>(
1033    info: WorkerInfo,
1034    source_factory: &S,
1035    transform: &mut F::Transform,
1036    credits: &Receiver<()>,
1037    results: &Sender<StreamCompletionFor<S, F, I, M>>,
1038    shutdown: &CancellationToken,
1039    run: &WorkerRunContext,
1040    active: &mut Option<(WorkerRunContext, Option<u64>, Option<u64>, bool)>,
1041    footprint: Option<StreamFootprint<S, F>>,
1042    ordered: bool,
1043) where
1044    S: WorkerSourceFactory,
1045    F: TransformFactory<S::Sample> + Send + Sync + 'static,
1046    F::Transform: Send + 'static,
1047    TransformOutput<S, F>: Send + 'static,
1048    TransformFailure<S, F>: Send + 'static,
1049    F::Error: Send + 'static,
1050    I: WorkerInit + Send + Sync + 'static,
1051    I::Error: Send + 'static,
1052    M: StreamMemoryPolicy,
1053{
1054    let generation_info = WorkerInfo::from_loader_seed(
1055        info.id,
1056        info.num_workers,
1057        run.loader_seed,
1058        info.rank,
1059        run.generation,
1060    )
1061    .expect("validated stream worker identity remains valid for every generation");
1062    let context = WorkerContext::new(
1063        generation_info,
1064        run.cancellation.clone(),
1065        run.deadline.clone(),
1066    );
1067    let mut source = match source_factory.create(context.clone()) {
1068        Ok(source) if context.check().is_ok() => source,
1069        Ok(source) => {
1070            drop(source);
1071            send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, false);
1072            return;
1073        }
1074        Err(error) => {
1075            send_stream_failure(
1076                results,
1077                shutdown,
1078                info.id,
1079                run.generation,
1080                None,
1081                None,
1082                false,
1083                StreamFailure::Source(error),
1084            );
1085            run.cancel();
1086            send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, false);
1087            return;
1088        }
1089    };
1090
1091    let mut last_sequence = None;
1092    loop {
1093        let acquired = select! {
1094            recv(credits) -> credit => credit.is_ok(),
1095            recv(run.cancellation.signal()) -> _ => false,
1096            recv(shutdown.signal()) -> _ => return,
1097        };
1098        if !acquired {
1099            drop(source);
1100            send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, false);
1101            return;
1102        }
1103        active.as_mut().expect("active stream generation").3 = true;
1104        if context.check().is_err() {
1105            drop(source);
1106            send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1107            active.as_mut().expect("active stream generation").3 = false;
1108            return;
1109        }
1110
1111        let record = match source.next() {
1112            Some(Ok(record)) => {
1113                if context.check().is_err() {
1114                    drop(source);
1115                    send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1116                    active.as_mut().expect("active stream generation").3 = false;
1117                    return;
1118                }
1119                record
1120            }
1121            None => {
1122                drop(source);
1123                send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1124                active.as_mut().expect("active stream generation").3 = false;
1125                return;
1126            }
1127            Some(Err(error)) => {
1128                drop(source);
1129                send_stream_failure(
1130                    results,
1131                    shutdown,
1132                    info.id,
1133                    run.generation,
1134                    None,
1135                    None,
1136                    true,
1137                    StreamFailure::Source(error),
1138                );
1139                active.as_mut().expect("active stream generation").3 = false;
1140                run.cancel();
1141                send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, false);
1142                return;
1143            }
1144        };
1145        let sequence = record.sequence.map(SequenceId::get);
1146        let logical_id = record.logical_id.get();
1147        {
1148            let active = active.as_mut().expect("active stream generation");
1149            active.1 = sequence;
1150            active.2 = Some(logical_id);
1151        }
1152        let task_context = TaskContext {
1153            loader_seed: run.loader_seed,
1154            epoch: run.epoch,
1155            rank: info.rank,
1156            logical_sample: logical_id,
1157            stage: 0,
1158            cancellation: run.cancellation.clone(),
1159            deadline: run.deadline.clone(),
1160        };
1161        let sample = match transform.transform(record.sample, &task_context) {
1162            Ok(sample) if context.check().is_ok() => sample,
1163            Ok(_) => {
1164                drop(source);
1165                send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1166                active.as_mut().expect("active stream generation").3 = false;
1167                return;
1168            }
1169            Err(error) => {
1170                drop(source);
1171                send_stream_failure(
1172                    results,
1173                    shutdown,
1174                    info.id,
1175                    run.generation,
1176                    sequence,
1177                    Some(logical_id),
1178                    true,
1179                    StreamFailure::Transform(error),
1180                );
1181                active.as_mut().expect("active stream generation").3 = false;
1182                run.cancel();
1183                send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, false);
1184                return;
1185            }
1186        };
1187        let permit = if let Some(footprint) = footprint {
1188            let actual = footprint(&sample);
1189            let budget = run
1190                .byte_budget
1191                .as_ref()
1192                .expect("enabled stream byte accounting has a generation budget");
1193            let acquired = if ordered {
1194                let Some(sequence) = sequence else {
1195                    drop(source);
1196                    send_stream_failure(
1197                        results,
1198                        shutdown,
1199                        info.id,
1200                        run.generation,
1201                        None,
1202                        Some(logical_id),
1203                        true,
1204                        StreamFailure::Memory(
1205                            M::protocol(
1206                                None,
1207                                "ordered byte-bounded stream record did not provide a sequence identifier".to_owned(),
1208                            )
1209                            .expect("enabled stream accounting has protocol failure storage"),
1210                        ),
1211                    );
1212                    active.as_mut().expect("active stream generation").3 = false;
1213                    run.cancel();
1214                    send_stream_end::<S, F, I, M>(
1215                        results,
1216                        shutdown,
1217                        info.id,
1218                        run.generation,
1219                        false,
1220                    );
1221                    return;
1222                };
1223                if last_sequence.is_some_and(|previous| sequence <= previous) {
1224                    drop(source);
1225                    send_stream_failure(
1226                        results,
1227                        shutdown,
1228                        info.id,
1229                        run.generation,
1230                        Some(sequence),
1231                        Some(logical_id),
1232                        true,
1233                        StreamFailure::Memory(
1234                            M::protocol(
1235                                Some(sequence),
1236                                format!(
1237                                    "worker {} emitted non-increasing sequence {sequence}",
1238                                    info.id
1239                                ),
1240                            )
1241                            .expect("enabled stream accounting has protocol failure storage"),
1242                        ),
1243                    );
1244                    active.as_mut().expect("active stream generation").3 = false;
1245                    run.cancel();
1246                    send_stream_end::<S, F, I, M>(
1247                        results,
1248                        shutdown,
1249                        info.id,
1250                        run.generation,
1251                        false,
1252                    );
1253                    return;
1254                }
1255                last_sequence = Some(sequence);
1256                if !send_stream_completion(
1257                    results,
1258                    shutdown,
1259                    Some(&run.cancellation),
1260                    StreamCompletion {
1261                        worker: info.id,
1262                        generation: run.generation,
1263                        sequence: Some(sequence),
1264                        logical_id: Some(logical_id),
1265                        holds_credit: false,
1266                        result: Ok(StreamMessage::Waiting(
1267                            M::waiting(sequence)
1268                                .expect("enabled stream accounting has waiter messages"),
1269                        )),
1270                    },
1271                ) {
1272                    drop(source);
1273                    send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1274                    active.as_mut().expect("active stream generation").3 = false;
1275                    return;
1276                }
1277                budget.acquire_ordered(sequence, actual)
1278            } else {
1279                budget.acquire(actual)
1280            };
1281            match acquired {
1282                Ok(permit) => Some(permit),
1283                Err(BudgetError::Cancelled) => {
1284                    drop(source);
1285                    send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1286                    active.as_mut().expect("active stream generation").3 = false;
1287                    return;
1288                }
1289                Err(BudgetError::Oversize { limit, actual }) => {
1290                    drop(source);
1291                    send_stream_failure(
1292                        results,
1293                        shutdown,
1294                        info.id,
1295                        run.generation,
1296                        sequence,
1297                        Some(logical_id),
1298                        true,
1299                        StreamFailure::Memory(
1300                            M::memory_limit(limit, actual)
1301                                .expect("enabled stream accounting has memory-limit storage"),
1302                        ),
1303                    );
1304                    active.as_mut().expect("active stream generation").3 = false;
1305                    run.cancel();
1306                    send_stream_end::<S, F, I, M>(
1307                        results,
1308                        shutdown,
1309                        info.id,
1310                        run.generation,
1311                        false,
1312                    );
1313                    return;
1314                }
1315                Err(BudgetError::SequenceAlreadyAdmitted) => {
1316                    drop(source);
1317                    send_stream_failure(
1318                        results,
1319                        shutdown,
1320                        info.id,
1321                        run.generation,
1322                        sequence,
1323                        Some(logical_id),
1324                        true,
1325                        StreamFailure::Memory(
1326                            M::protocol(
1327                                sequence,
1328                                "sequence was already admitted to the byte budget".to_owned(),
1329                            )
1330                            .expect("enabled stream accounting has protocol failure storage"),
1331                        ),
1332                    );
1333                    active.as_mut().expect("active stream generation").3 = false;
1334                    run.cancel();
1335                    send_stream_end::<S, F, I, M>(
1336                        results,
1337                        shutdown,
1338                        info.id,
1339                        run.generation,
1340                        false,
1341                    );
1342                    return;
1343                }
1344            }
1345        } else {
1346            None
1347        };
1348        let sent = send_stream_completion(
1349            results,
1350            shutdown,
1351            Some(&run.cancellation),
1352            StreamCompletion {
1353                worker: info.id,
1354                generation: run.generation,
1355                sequence,
1356                logical_id: Some(logical_id),
1357                holds_credit: true,
1358                result: Ok(StreamMessage::Record(
1359                    WorkerRecord {
1360                        sequence: record.sequence,
1361                        logical_id: record.logical_id,
1362                        sample,
1363                    },
1364                    M::permit(permit),
1365                )),
1366            },
1367        );
1368        if sent {
1369            let active = active.as_mut().expect("active stream generation");
1370            active.1 = None;
1371            active.2 = None;
1372            active.3 = false;
1373        } else {
1374            drop(source);
1375            send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1376            active.as_mut().expect("active stream generation").3 = false;
1377            return;
1378        }
1379    }
1380}
1381
1382fn send_stream_end<S, F, I, M>(
1383    results: &Sender<StreamCompletionFor<S, F, I, M>>,
1384    shutdown: &CancellationToken,
1385    worker: usize,
1386    generation: u64,
1387    holds_credit: bool,
1388) where
1389    S: WorkerSourceFactory,
1390    F: TransformFactory<S::Sample>,
1391    I: WorkerInit,
1392    M: StreamMemoryPolicy,
1393{
1394    let _ = send_stream_completion(
1395        results,
1396        shutdown,
1397        None,
1398        StreamCompletion {
1399            worker,
1400            generation,
1401            sequence: None,
1402            logical_id: None,
1403            holds_credit,
1404            result: Ok(StreamMessage::End),
1405        },
1406    );
1407}
1408
1409#[allow(clippy::too_many_arguments)]
1410fn send_stream_failure<T, P, W, ME, SE, TE, FE, IE>(
1411    results: &StreamCompletionSender<T, P, W, ME, SE, TE, FE, IE>,
1412    shutdown: &CancellationToken,
1413    worker: usize,
1414    generation: u64,
1415    sequence: Option<u64>,
1416    logical_id: Option<u64>,
1417    holds_credit: bool,
1418    failure: StreamFailure<SE, TE, FE, IE, ME>,
1419) {
1420    let _ = send_stream_completion(
1421        results,
1422        shutdown,
1423        None,
1424        StreamCompletion {
1425            worker,
1426            generation,
1427            sequence,
1428            logical_id,
1429            holds_credit,
1430            result: Err(failure),
1431        },
1432    );
1433}
1434
1435fn send_stream_completion<T, P, W, ME, SE, TE, FE, IE>(
1436    results: &StreamCompletionSender<T, P, W, ME, SE, TE, FE, IE>,
1437    shutdown: &CancellationToken,
1438    cancellation: Option<&CancellationToken>,
1439    completion: StreamCompletion<T, P, W, ME, SE, TE, FE, IE>,
1440) -> bool {
1441    if let Some(cancellation) = cancellation {
1442        select! {
1443            send(results, completion) -> result => result.is_ok(),
1444            recv(cancellation.signal()) -> _ => false,
1445            recv(shutdown.signal()) -> _ => false,
1446        }
1447    } else {
1448        select! {
1449            send(results, completion) -> result => result.is_ok(),
1450            recv(shutdown.signal()) -> _ => false,
1451        }
1452    }
1453}
1454
1455#[derive(Clone, Copy)]
1456struct StreamConfiguration {
1457    workers: usize,
1458    batch_size: usize,
1459    drop_last: bool,
1460    prefetch_factor: usize,
1461    ordered: bool,
1462    persistent_workers: bool,
1463    timeout: Option<Duration>,
1464    loader_seed: u64,
1465    epoch: u64,
1466    rank: usize,
1467    prefetch_bytes: Option<NonZeroUsize>,
1468    pin_memory: StreamPinRequest,
1469}
1470
1471#[derive(Clone, Copy)]
1472enum StreamPinRequest {
1473    Disabled,
1474    Auto,
1475    Explicit(Device),
1476}
1477
1478impl Default for StreamConfiguration {
1479    fn default() -> Self {
1480        Self {
1481            workers: 1,
1482            batch_size: 1,
1483            drop_last: false,
1484            prefetch_factor: 2,
1485            ordered: true,
1486            persistent_workers: false,
1487            timeout: None,
1488            loader_seed: 0,
1489            epoch: 0,
1490            rank: 0,
1491            prefetch_bytes: None,
1492            pin_memory: StreamPinRequest::Disabled,
1493        }
1494    }
1495}
1496
1497/// Merge lazy worker-owned streams into batches for training or inference.
1498///
1499/// Use this when records are read sequentially from independent shards rather
1500/// than fetched by index. Each [`WorkerSourceFactory`] call returns one owned
1501/// iterator. Workers read and transform records; the calling thread merges and
1502/// collates them. For a single existing iterator, [`crate::batches`] is simpler.
1503///
1504/// # Read disjoint shards without collecting them first
1505///
1506/// This factory assigns every `workers`-th record to one worker. Together, the
1507/// shards provide contiguous global sequence IDs, so the loader restores input
1508/// order even if a later record finishes first. Five records produce two full
1509/// batches and one final short batch across the entire source.
1510///
1511/// ```
1512/// use std::{convert::Infallible, iter::StepBy, ops::Range};
1513/// use rusttorch_data::{
1514///     LogicalSampleId, SequenceId, StreamDataLoaderBuilder, VecCollate,
1515///     WorkerContext, WorkerRecord, WorkerSourceFactory,
1516/// };
1517/// struct Rows(usize);
1518/// struct Shard(StepBy<Range<usize>>);
1519/// impl Iterator for Shard {
1520///     type Item = Result<WorkerRecord<usize>, Infallible>;
1521///     fn next(&mut self) -> Option<Self::Item> {
1522///         self.0.next().map(|row| Ok(WorkerRecord {
1523///             sequence: Some(SequenceId::new(row as u64)),
1524///             logical_id: LogicalSampleId::new(row as u64),
1525///             sample: row,
1526///         }))
1527///     }
1528/// }
1529/// impl WorkerSourceFactory for Rows {
1530///     type Sample = usize;
1531///     type Error = Infallible;
1532///     type Source = Shard;
1533///     fn create(&self, worker: WorkerContext) -> Result<Shard, Infallible> {
1534///         Ok(Shard((worker.info.id..self.0).step_by(worker.info.num_workers)))
1535///     }
1536///     fn exact_len(&self) -> Option<usize> { Some(self.0) }
1537/// }
1538/// fn main() -> Result<(), Box<dyn std::error::Error>> {
1539///     let mut loader = StreamDataLoaderBuilder::new(Rows(5))
1540///         .workers(2).prefetch_factor(2).batch_size(2)
1541///         .collate(VecCollate).build()?;
1542///     let batches = loader.iter().collect::<Result<Vec<_>, _>>()?;
1543///     assert_eq!(batches, [vec![0, 1], vec![2, 3], vec![4]]);
1544///     Ok(())
1545/// }
1546/// ```
1547///
1548/// # Select ordering and resource bounds
1549///
1550/// - Defaults: one worker, ordered delivery, batch size one, keeping the final
1551///   tail, and prefetch factor two. Unlike map loaders, workers must be positive.
1552/// - [`Self::drop_last`] applies after all shards are merged: uneven shards do
1553///   not each lose a separate tail.
1554/// - [`Self::prefetch_factor`] limits outstanding **records** per worker. Ordered
1555///   IDs must start at zero without gaps or duplicates. If higher IDs fill the
1556///   whole window before the next ID can be reached, iteration returns a protocol
1557///   error; adjust sharding or the factor to keep that next record reachable.
1558/// - [`Self::ordered(false)`](Self::ordered) permits completion-order delivery
1559///   and omitted sequence IDs; logical sample IDs remain required.
1560/// - [`Self::prefetch_bytes`] additionally limits transformed payload bytes.
1561///   Ordered shards must then increase strictly in sequence order. The active
1562///   collation batch and allocations inside your callbacks are outside this cap.
1563///
1564/// `iter()` opens fresh shards for another pass. [`StreamDataLoader::set_epoch`]
1565/// changes sample-transform randomness for the next pass. Persistent workers
1566/// reuse threads and transforms, while each pass still opens a fresh source.
1567/// Dropping an iterator cancels its work and waits for it to stop; dropping the
1568/// owner joins persistent threads too. Blocking sources must cooperate with
1569/// cancellation if prompt shutdown matters.
1570///
1571/// For restartable streams, implement [`crate::CheckpointableSource`] and
1572/// [`crate::CheckpointSourceFactory`], then use [`Self::checkpointable`].
1573/// [`ExactStreamDataLoader`] contains a complete JSON checkpoint/restore example.
1574pub struct StreamDataLoaderBuilder<
1575    S,
1576    C = DefaultCollator,
1577    F = IdentityTransformFactory,
1578    I = NoWorkerInit,
1579    M = MemoryDisabled,
1580    N = PinDisabled,
1581    K = crate::CheckpointDisabled,
1582> {
1583    factory: S,
1584    collator: C,
1585    transform_factory: F,
1586    worker_init: I,
1587    configuration: StreamConfiguration,
1588    checkpoint: K,
1589    states: PhantomData<(M, N)>,
1590}
1591
1592impl<S> StreamDataLoaderBuilder<S>
1593where
1594    S: WorkerSourceFactory,
1595{
1596    /// Starts a builder with one worker, ordered delivery, batch size one,
1597    /// prefetch factor two, no timeout, and no persistence.
1598    pub fn new(factory: S) -> Self {
1599        Self {
1600            factory,
1601            collator: DefaultCollator,
1602            transform_factory: IdentityTransformFactory,
1603            worker_init: NoWorkerInit,
1604            configuration: StreamConfiguration::default(),
1605            checkpoint: crate::CheckpointDisabled,
1606            states: PhantomData,
1607        }
1608    }
1609}
1610
1611impl<S, C, F, I, M, N, K> StreamDataLoaderBuilder<S, C, F, I, M, N, K> {
1612    /// Sets the positive stream worker count.
1613    pub fn workers(mut self, workers: usize) -> Self {
1614        self.configuration.workers = workers;
1615        self
1616    }
1617
1618    /// Sets the global coordinator batch size.
1619    pub fn batch_size(mut self, batch_size: usize) -> Self {
1620        self.configuration.batch_size = batch_size;
1621        self
1622    }
1623
1624    /// Selects whether the one final global short batch is omitted.
1625    pub fn drop_last(mut self, drop_last: bool) -> Self {
1626        self.configuration.drop_last = drop_last;
1627        self
1628    }
1629
1630    /// Sets bounded unpublished records permitted per worker.
1631    ///
1632    /// In ordered mode, filling the complete global reassembly window with
1633    /// higher IDs before the next ID is observable is a stream protocol error.
1634    pub fn prefetch_factor(mut self, factor: usize) -> Self {
1635        self.configuration.prefetch_factor = factor;
1636        self
1637    }
1638
1639    /// Enables a nonzero byte budget for final post-transform records.
1640    ///
1641    /// Ordered byte-bounded shards must emit strictly increasing sequence IDs.
1642    /// The coordinator tracks one bounded front waiter per shard and reports a
1643    /// protocol error when every active shard has advanced beyond a missing ID.
1644    pub fn prefetch_bytes(
1645        mut self,
1646        limit: NonZeroUsize,
1647    ) -> StreamDataLoaderBuilder<S, C, F, I, MemoryEnabled, N, K> {
1648        self.configuration.prefetch_bytes = Some(limit);
1649        StreamDataLoaderBuilder {
1650            factory: self.factory,
1651            collator: self.collator,
1652            transform_factory: self.transform_factory,
1653            worker_init: self.worker_init,
1654            configuration: self.configuration,
1655            checkpoint: self.checkpoint,
1656            states: PhantomData,
1657        }
1658    }
1659
1660    /// Enables automatic recursive pinning for CUDA device zero when available.
1661    pub fn pin_memory(mut self) -> StreamDataLoaderBuilder<S, C, F, I, M, PinEnabled<Auto>, K> {
1662        self.configuration.pin_memory = StreamPinRequest::Auto;
1663        StreamDataLoaderBuilder {
1664            factory: self.factory,
1665            collator: self.collator,
1666            transform_factory: self.transform_factory,
1667            worker_init: self.worker_init,
1668            configuration: self.configuration,
1669            checkpoint: self.checkpoint,
1670            states: PhantomData,
1671        }
1672    }
1673
1674    /// Enables recursive pinning for one explicit, available CUDA device.
1675    pub fn pin_memory_for(
1676        mut self,
1677        device: Device,
1678    ) -> StreamDataLoaderBuilder<S, C, F, I, M, PinEnabled<Explicit>, K> {
1679        self.configuration.pin_memory = StreamPinRequest::Explicit(device);
1680        StreamDataLoaderBuilder {
1681            factory: self.factory,
1682            collator: self.collator,
1683            transform_factory: self.transform_factory,
1684            worker_init: self.worker_init,
1685            configuration: self.configuration,
1686            checkpoint: self.checkpoint,
1687            states: PhantomData,
1688        }
1689    }
1690
1691    /// Selects globally sequenced or completion-order delivery.
1692    pub fn ordered(mut self, ordered: bool) -> Self {
1693        self.configuration.ordered = ordered;
1694        self
1695    }
1696
1697    /// Selects globally sequenced or completion-order delivery.
1698    pub fn in_order(self, in_order: bool) -> Self {
1699        self.ordered(in_order)
1700    }
1701
1702    /// Sets the deterministic loader seed.
1703    pub fn seed(mut self, seed: u64) -> Self {
1704        self.configuration.loader_seed = seed;
1705        self
1706    }
1707
1708    /// Sets the initial stream epoch.
1709    pub fn epoch(mut self, epoch: u64) -> Self {
1710        self.configuration.epoch = epoch;
1711        self
1712    }
1713
1714    /// Sets the distributed rank included in worker and task contexts.
1715    pub fn rank(mut self, rank: usize) -> Self {
1716        self.configuration.rank = rank;
1717        self
1718    }
1719
1720    /// Sets the cooperative timeout for each blocking iterator call.
1721    pub fn timeout(mut self, timeout: Duration) -> Self {
1722        self.configuration.timeout = (!timeout.is_zero()).then_some(timeout);
1723        self
1724    }
1725
1726    /// Selects loader-owned persistent worker threads.
1727    pub fn persistent_workers(mut self, persistent: bool) -> Self {
1728        self.configuration.persistent_workers = persistent;
1729        self
1730    }
1731
1732    /// Replaces coordinator collation.
1733    pub fn collate<C2>(self, collator: C2) -> StreamDataLoaderBuilder<S, C2, F, I, M, N, K> {
1734        StreamDataLoaderBuilder {
1735            factory: self.factory,
1736            collator,
1737            transform_factory: self.transform_factory,
1738            worker_init: self.worker_init,
1739            configuration: self.configuration,
1740            checkpoint: self.checkpoint,
1741            states: PhantomData,
1742        }
1743    }
1744
1745    /// Clones one transform template for each worker lifecycle.
1746    pub fn transform<T>(
1747        self,
1748        transform: T,
1749    ) -> StreamDataLoaderBuilder<S, C, CloneTransformFactory<T>, I, M, N, K> {
1750        StreamDataLoaderBuilder {
1751            factory: self.factory,
1752            collator: self.collator,
1753            transform_factory: CloneTransformFactory::new(transform),
1754            worker_init: self.worker_init,
1755            configuration: self.configuration,
1756            checkpoint: self.checkpoint,
1757            states: PhantomData,
1758        }
1759    }
1760
1761    /// Replaces worker transform construction.
1762    pub fn transform_factory<F2>(
1763        self,
1764        factory: F2,
1765    ) -> StreamDataLoaderBuilder<S, C, F2, I, M, N, K> {
1766        StreamDataLoaderBuilder {
1767            factory: self.factory,
1768            collator: self.collator,
1769            transform_factory: factory,
1770            worker_init: self.worker_init,
1771            configuration: self.configuration,
1772            checkpoint: self.checkpoint,
1773            states: PhantomData,
1774        }
1775    }
1776
1777    /// Replaces worker initialization.
1778    pub fn worker_init<I2>(self, worker_init: I2) -> StreamDataLoaderBuilder<S, C, F, I2, M, N, K> {
1779        StreamDataLoaderBuilder {
1780            factory: self.factory,
1781            collator: self.collator,
1782            transform_factory: self.transform_factory,
1783            worker_init,
1784            configuration: self.configuration,
1785            checkpoint: self.checkpoint,
1786            states: PhantomData,
1787        }
1788    }
1789
1790    /// Validates configuration and constructs the stream owner.
1791    fn build_inner(
1792        self,
1793        footprint: Option<StreamFootprint<S, F>>,
1794    ) -> Result<StreamDataLoader<S, C, F, I, M, N>>
1795    where
1796        S: WorkerSourceFactory,
1797        F: TransformFactory<S::Sample>,
1798        C: Collate<TransformOutput<S, F>>,
1799        I: WorkerInit,
1800        M: StreamMemoryPolicy,
1801    {
1802        let pin_memory_status = resolve_stream_pin(self.configuration.pin_memory)?;
1803        if self.configuration.workers == 0 {
1804            return Err(invalid_configuration(
1805                "workers",
1806                "stream workers must be positive; use batches or batches_with_collate for an ordinary iterator",
1807            ));
1808        }
1809        if self.configuration.batch_size == 0 {
1810            return Err(invalid_configuration(
1811                "batch_size",
1812                "must be greater than zero",
1813            ));
1814        }
1815        if self.configuration.prefetch_factor == 0 {
1816            return Err(invalid_configuration(
1817                "prefetch_factor",
1818                "must be greater than zero",
1819            ));
1820        }
1821        if self
1822            .configuration
1823            .timeout
1824            .is_some_and(|timeout| !Deadline::can_represent(timeout))
1825        {
1826            return Err(invalid_configuration(
1827                "timeout",
1828                "exceeds the platform monotonic clock range",
1829            ));
1830        }
1831        let outstanding_capacity = self
1832            .configuration
1833            .workers
1834            .checked_mul(self.configuration.prefetch_factor)
1835            .ok_or_else(|| {
1836                invalid_configuration(
1837                    "prefetch_factor",
1838                    "workers multiplied by prefetch_factor exceeds usize",
1839                )
1840            })?;
1841        validate_stream_capacity::<S, F, I, M>(
1842            self.configuration.workers,
1843            self.configuration.prefetch_factor,
1844            outstanding_capacity,
1845            self.configuration.batch_size,
1846            usize::from(self.configuration.ordered) * outstanding_capacity,
1847        )?;
1848        let mut ordered_reassembly = Vec::new();
1849        if self.configuration.ordered {
1850            ordered_reassembly
1851                .try_reserve_exact(outstanding_capacity)
1852                .map_err(|error| {
1853                    capacity_error(format!(
1854                        "ordered stream reassembly capacity is unavailable: {error}"
1855                    ))
1856                })?;
1857        }
1858        validate_stream_capacity::<S, F, I, M>(
1859            self.configuration.workers,
1860            self.configuration.prefetch_factor,
1861            outstanding_capacity,
1862            self.configuration.batch_size,
1863            ordered_reassembly.capacity(),
1864        )?;
1865        let exact_len = self.factory.exact_len();
1866        Ok(StreamDataLoader {
1867            factory: Arc::new(self.factory),
1868            collator: self.collator,
1869            transform_factory: Arc::new(self.transform_factory),
1870            worker_init: Arc::new(self.worker_init),
1871            configuration: self.configuration,
1872            outstanding_capacity,
1873            exact_len,
1874            next_generation: 0,
1875            persistent_pool: None,
1876            ordered_reassembly: Mutex::new(ordered_reassembly),
1877            footprint,
1878            pin_memory_status,
1879            policies: PhantomData,
1880        })
1881    }
1882}
1883
1884fn stream_resident_bytes<T: MemoryFootprint>(value: &T) -> usize {
1885    value.resident_bytes()
1886}
1887
1888fn resolve_stream_pin(request: StreamPinRequest) -> Result<PinMemoryStatus> {
1889    match request {
1890        StreamPinRequest::Disabled => Ok(PinMemoryStatus::Disabled),
1891        StreamPinRequest::Auto => {
1892            let capabilities = available_devices();
1893            if capabilities.cuda && capabilities.cuda_device_count > 0 {
1894                Ok(PinMemoryStatus::Enabled(Device::Cuda(0)))
1895            } else {
1896                Ok(PinMemoryStatus::DisabledNoAccelerator)
1897            }
1898        }
1899        StreamPinRequest::Explicit(Device::Cuda(index)) => {
1900            let capabilities = available_devices();
1901            if capabilities.cuda && index < capabilities.cuda_device_count {
1902                Ok(PinMemoryStatus::Enabled(Device::Cuda(index)))
1903            } else {
1904                Err(invalid_configuration(
1905                    "pin_memory",
1906                    format!(
1907                        "CUDA device {index} was requested, but the linked runtime exposes {} available CUDA device(s)",
1908                        capabilities.cuda_device_count
1909                    ),
1910                ))
1911            }
1912        }
1913        StreamPinRequest::Explicit(device) => Err(invalid_configuration(
1914            "pin_memory",
1915            format!("only an available CUDA device can back pinned host memory, got {device:?}"),
1916        )),
1917    }
1918}
1919
1920impl<S, C, F, I> StreamDataLoaderBuilder<S, C, F, I, MemoryDisabled, PinDisabled> {
1921    /// Validates configuration and builds an item-bounded, unpinned stream.
1922    pub fn build(self) -> Result<StreamDataLoader<S, C, F, I, MemoryDisabled, PinDisabled>>
1923    where
1924        S: WorkerSourceFactory,
1925        F: TransformFactory<S::Sample>,
1926        C: Collate<TransformOutput<S, F>>,
1927        I: WorkerInit,
1928    {
1929        self.build_inner(None)
1930    }
1931}
1932
1933impl<S, C, F, I> StreamDataLoaderBuilder<S, C, F, I, MemoryEnabled, PinDisabled> {
1934    /// Validates configuration and builds a byte-bounded, unpinned stream.
1935    pub fn build(self) -> Result<StreamDataLoader<S, C, F, I, MemoryEnabled, PinDisabled>>
1936    where
1937        S: WorkerSourceFactory,
1938        F: TransformFactory<S::Sample>,
1939        TransformOutput<S, F>: MemoryFootprint,
1940        C: Collate<TransformOutput<S, F>>,
1941        I: WorkerInit,
1942    {
1943        self.build_inner(Some(stream_resident_bytes::<TransformOutput<S, F>>))
1944    }
1945}
1946
1947impl<S, C, F, I, Q> StreamDataLoaderBuilder<S, C, F, I, MemoryDisabled, PinEnabled<Q>> {
1948    /// Validates configuration and builds an item-bounded pinned stream.
1949    pub fn build(self) -> Result<StreamDataLoader<S, C, F, I, MemoryDisabled, PinEnabled<Q>>>
1950    where
1951        S: WorkerSourceFactory,
1952        F: TransformFactory<S::Sample>,
1953        C: Collate<TransformOutput<S, F>>,
1954        C::Batch: PinMemory,
1955        I: WorkerInit,
1956    {
1957        self.build_inner(None)
1958    }
1959}
1960
1961impl<S, C, F, I, Q> StreamDataLoaderBuilder<S, C, F, I, MemoryEnabled, PinEnabled<Q>> {
1962    /// Validates configuration and builds a byte-bounded pinned stream.
1963    pub fn build(self) -> Result<StreamDataLoader<S, C, F, I, MemoryEnabled, PinEnabled<Q>>>
1964    where
1965        S: WorkerSourceFactory,
1966        F: TransformFactory<S::Sample>,
1967        TransformOutput<S, F>: MemoryFootprint,
1968        C: Collate<TransformOutput<S, F>>,
1969        C::Batch: PinMemory,
1970        I: WorkerInit,
1971    {
1972        self.build_inner(Some(stream_resident_bytes::<TransformOutput<S, F>>))
1973    }
1974}
1975
1976/// Reusable owner of a sharded input pipeline.
1977///
1978/// Create it with [`StreamDataLoaderBuilder`], whose example reads lazy disjoint
1979/// shards into global batches. Call [`Self::iter`] for each pass and
1980/// [`Self::set_epoch`] before the next epoch when transforms use task randomness.
1981/// [`Self::len`] is the global batch count when the factory knows its exact length.
1982///
1983/// Each pass creates new source iterators. Persistent mode keeps worker threads
1984/// and transforms across passes; dropping this owner shuts them down and joins
1985/// them. Use [`ExactStreamDataLoader`] when you need to retain a source cursor
1986/// across application restarts.
1987#[allow(private_bounds)]
1988pub struct StreamDataLoader<
1989    S,
1990    C,
1991    F = IdentityTransformFactory,
1992    I = NoWorkerInit,
1993    M = MemoryDisabled,
1994    N = PinDisabled,
1995> where
1996    S: WorkerSourceFactory,
1997    F: TransformFactory<S::Sample>,
1998    C: Collate<TransformOutput<S, F>>,
1999    I: WorkerInit,
2000    M: StreamMemoryPolicy,
2001{
2002    factory: Arc<S>,
2003    collator: C,
2004    transform_factory: Arc<F>,
2005    worker_init: Arc<I>,
2006    configuration: StreamConfiguration,
2007    outstanding_capacity: usize,
2008    exact_len: Option<usize>,
2009    next_generation: u64,
2010    persistent_pool: Option<StreamWorkerPool<S, F, I, M>>,
2011    // `iter` has exclusive loader access; this wrapper preserves `Sync` for
2012    // output that is `Send` but not `Sync` without runtime contention.
2013    ordered_reassembly: RetainedReassembly<TransformOutput<S, F>, M::Permit>,
2014    footprint: Option<StreamFootprint<S, F>>,
2015    pin_memory_status: PinMemoryStatus,
2016    policies: PhantomData<(M, N)>,
2017}
2018
2019#[allow(private_bounds)]
2020impl<S, C, F, I, M, N> StreamDataLoader<S, C, F, I, M, N>
2021where
2022    S: WorkerSourceFactory,
2023    F: TransformFactory<S::Sample>,
2024    C: Collate<TransformOutput<S, F>>,
2025    I: WorkerInit,
2026    M: StreamMemoryPolicy,
2027{
2028    /// Returns the exact global batch count when the factory is sized.
2029    pub fn len(&self) -> Option<usize> {
2030        self.exact_len.map(|length| {
2031            let complete = length / self.configuration.batch_size;
2032            complete
2033                + usize::from(
2034                    !self.configuration.drop_last
2035                        && !length.is_multiple_of(self.configuration.batch_size),
2036                )
2037        })
2038    }
2039
2040    /// Returns whether the sized stream yields no batches.
2041    pub fn is_empty(&self) -> Option<bool> {
2042        self.len().map(|length| length == 0)
2043    }
2044
2045    /// Returns the configured worker count.
2046    pub fn workers(&self) -> usize {
2047        self.configuration.workers
2048    }
2049
2050    /// Returns the effective records-prefetched-per-worker factor.
2051    pub fn effective_prefetch_factor(&self) -> NonZeroUsize {
2052        NonZeroUsize::new(self.configuration.prefetch_factor)
2053            .expect("validated prefetch factor is nonzero")
2054    }
2055
2056    /// Returns the effective post-transform record byte budget.
2057    pub fn effective_prefetch_bytes(&self) -> Option<NonZeroUsize> {
2058        self.configuration.prefetch_bytes
2059    }
2060
2061    /// Returns whether recursive batch pinning was requested.
2062    pub fn pin_memory_enabled(&self) -> bool {
2063        !matches!(self.pin_memory_status, PinMemoryStatus::Disabled)
2064    }
2065
2066    /// Returns the effective recursive pinning behavior.
2067    pub fn pin_memory_status(&self) -> PinMemoryStatus {
2068        self.pin_memory_status
2069    }
2070
2071    /// Returns whether ordered delivery is enabled.
2072    pub fn is_ordered(&self) -> bool {
2073        self.configuration.ordered
2074    }
2075
2076    /// Returns the configured epoch.
2077    pub fn epoch(&self) -> u64 {
2078        self.configuration.epoch
2079    }
2080
2081    /// Selects the epoch used by future iterator generations.
2082    pub fn set_epoch(&mut self, epoch: u64) {
2083        self.configuration.epoch = epoch;
2084    }
2085}
2086
2087struct BufferedRecord<T, P> {
2088    worker: usize,
2089    record: WorkerRecord<T>,
2090    permit: P,
2091}
2092
2093enum StreamIteratorPool<'a, S, F, I, M>
2094where
2095    S: WorkerSourceFactory,
2096    F: TransformFactory<S::Sample>,
2097    I: WorkerInit,
2098    M: StreamMemoryPolicy,
2099{
2100    Owned(StreamWorkerPool<S, F, I, M>),
2101    Persistent(&'a mut StreamWorkerPool<S, F, I, M>),
2102}
2103
2104impl<S, F, I, M> StreamIteratorPool<'_, S, F, I, M>
2105where
2106    S: WorkerSourceFactory,
2107    F: TransformFactory<S::Sample>,
2108    I: WorkerInit,
2109    M: StreamMemoryPolicy,
2110{
2111    fn pool(&self) -> &StreamWorkerPool<S, F, I, M> {
2112        match self {
2113            Self::Owned(pool) => pool,
2114            Self::Persistent(pool) => pool,
2115        }
2116    }
2117
2118    fn pool_mut(&mut self) -> &mut StreamWorkerPool<S, F, I, M> {
2119        match self {
2120            Self::Owned(pool) => pool,
2121            Self::Persistent(pool) => pool,
2122        }
2123    }
2124
2125    fn is_persistent(&self) -> bool {
2126        matches!(self, Self::Persistent(_))
2127    }
2128}
2129
2130#[allow(private_bounds)]
2131impl<S, C, F, I, M, N> StreamDataLoader<S, C, F, I, M, N>
2132where
2133    S: WorkerSourceFactory,
2134    F: TransformFactory<S::Sample> + Send + Sync + 'static,
2135    F::Transform: Send + 'static,
2136    TransformOutput<S, F>: Send + 'static,
2137    TransformFailure<S, F>: Send + 'static,
2138    F::Error: Send + 'static,
2139    C: Collate<TransformOutput<S, F>>,
2140    I: WorkerInit + Send + Sync + 'static,
2141    I::Error: Send + 'static,
2142    M: StreamMemoryPolicy + Send + 'static,
2143    N: StreamPinPolicy<S, C, F>,
2144{
2145    /// Starts a fresh explicitly sharded worker generation.
2146    pub fn iter(&mut self) -> StreamLoaderIter<'_, S, C, F, I, M, N> {
2147        let ordered_reassembly = match self.ordered_reassembly.get_mut() {
2148            Ok(reassembly) => reassembly,
2149            Err(poisoned) => poisoned.into_inner(),
2150        };
2151        ordered_reassembly.clear();
2152        let reassembly_allocation_slots = ordered_reassembly.capacity();
2153        let generation = self.next_generation;
2154        let run_context = WorkerRunContext::new(
2155            generation,
2156            self.configuration.loader_seed,
2157            self.configuration.epoch,
2158        )
2159        .with_byte_budget(
2160            self.configuration
2161                .prefetch_bytes
2162                .map(|limit| ByteBudget::new(limit.get())),
2163        );
2164        let mut pending_error = None;
2165        if let Some(next) = generation.checked_add(1) {
2166            self.next_generation = next;
2167        } else {
2168            pending_error = Some(LoaderError::Configuration(invalid_configuration(
2169                "workers",
2170                "stream iterator generation overflowed",
2171            )));
2172        }
2173
2174        let mut partial = Vec::new();
2175        if partial
2176            .try_reserve_exact(self.configuration.batch_size)
2177            .is_err()
2178        {
2179            pending_error = Some(LoaderError::Configuration(invalid_configuration(
2180                "batch_size",
2181                "stream batch allocation is unavailable",
2182            )));
2183        }
2184
2185        let front_waiters = match M::new_waiters(self.configuration.workers) {
2186            Ok(waiters) => waiters,
2187            Err(error) => {
2188                pending_error = Some(LoaderError::Configuration(error));
2189                M::new_waiters(0).expect("zero waiter state is always allocatable")
2190            }
2191        };
2192
2193        let mut pool = None;
2194        if pending_error.is_none() {
2195            if self.configuration.persistent_workers {
2196                if self.persistent_pool.is_none() {
2197                    match StreamWorkerPool::new(
2198                        Arc::clone(&self.factory),
2199                        Arc::clone(&self.transform_factory),
2200                        Arc::clone(&self.worker_init),
2201                        self.configuration,
2202                        generation,
2203                        self.outstanding_capacity,
2204                        reassembly_allocation_slots,
2205                        self.footprint,
2206                    ) {
2207                        Ok(created) => self.persistent_pool = Some(created),
2208                        Err(error) => pending_error = Some(LoaderError::Configuration(error)),
2209                    }
2210                }
2211                if pending_error.is_none() {
2212                    let persistent = self
2213                        .persistent_pool
2214                        .as_mut()
2215                        .expect("persistent stream pool was created");
2216                    if persistent.start_generation(run_context.clone()).is_err() {
2217                        pending_error = Some(LoaderError::ChannelClosed { batch: 0 });
2218                    } else {
2219                        pool = Some(StreamIteratorPool::Persistent(persistent));
2220                    }
2221                }
2222            } else {
2223                match StreamWorkerPool::new(
2224                    Arc::clone(&self.factory),
2225                    Arc::clone(&self.transform_factory),
2226                    Arc::clone(&self.worker_init),
2227                    self.configuration,
2228                    generation,
2229                    self.outstanding_capacity,
2230                    reassembly_allocation_slots,
2231                    self.footprint,
2232                ) {
2233                    Ok(mut created) => {
2234                        if created.start_generation(run_context.clone()).is_err() {
2235                            pending_error = Some(LoaderError::ChannelClosed { batch: 0 });
2236                        } else {
2237                            pool = Some(StreamIteratorPool::Owned(created));
2238                        }
2239                    }
2240                    Err(error) => pending_error = Some(LoaderError::Configuration(error)),
2241                }
2242            }
2243        }
2244
2245        StreamLoaderIter {
2246            collator: &mut self.collator,
2247            pool,
2248            run_context,
2249            generation,
2250            ordered: self.configuration.ordered,
2251            timeout: self.configuration.timeout,
2252            batch_size: self.configuration.batch_size,
2253            drop_last: self.configuration.drop_last,
2254            expected_records: self.exact_len,
2255            reassembly_capacity: self.outstanding_capacity,
2256            completed: ordered_reassembly,
2257            partial,
2258            next_sequence: 0,
2259            next_batch: 0,
2260            pending_error,
2261            source_complete: false,
2262            exhausted: false,
2263            front_waiters,
2264            pin_memory_status: self.pin_memory_status,
2265            marker: PhantomData,
2266        }
2267    }
2268}
2269
2270/// One pass over a [`StreamDataLoader`], yielding globally collated batches.
2271///
2272/// Read each `Result` before updating the model so a source, transform or
2273/// collation failure is handled at a visible batch boundary. Dropping this
2274/// iterator cancels outstanding reads and waits for active work. It does not
2275/// provide checkpointing; [`ExactStreamLoaderIter`] is the resumable variant.
2276#[allow(private_bounds)]
2277pub struct StreamLoaderIter<'a, S, C, F, I, M = MemoryDisabled, N = PinDisabled>
2278where
2279    S: WorkerSourceFactory,
2280    F: TransformFactory<S::Sample>,
2281    C: Collate<TransformOutput<S, F>>,
2282    I: WorkerInit,
2283    M: StreamMemoryPolicy,
2284{
2285    collator: &'a mut C,
2286    pool: Option<StreamIteratorPool<'a, S, F, I, M>>,
2287    run_context: WorkerRunContext,
2288    generation: u64,
2289    ordered: bool,
2290    timeout: Option<Duration>,
2291    batch_size: usize,
2292    drop_last: bool,
2293    expected_records: Option<usize>,
2294    reassembly_capacity: usize,
2295    completed: &'a mut Vec<ReassemblyEntry<TransformOutput<S, F>, M::Permit>>,
2296    partial: Vec<TransformOutput<S, F>>,
2297    next_sequence: u64,
2298    next_batch: u64,
2299    pending_error: Option<StreamLoaderError<S, C, F, I>>,
2300    source_complete: bool,
2301    exhausted: bool,
2302    front_waiters: M::Waiters,
2303    pin_memory_status: PinMemoryStatus,
2304    marker: PhantomData<&'a mut (S, C, F, I, M, N)>,
2305}
2306
2307#[allow(private_bounds)]
2308impl<S, C, F, I, M, N> StreamLoaderIter<'_, S, C, F, I, M, N>
2309where
2310    S: WorkerSourceFactory,
2311    F: TransformFactory<S::Sample> + Send + Sync + 'static,
2312    F::Transform: Send + 'static,
2313    TransformOutput<S, F>: Send + 'static,
2314    TransformFailure<S, F>: Send + 'static,
2315    F::Error: Send + 'static,
2316    C: Collate<TransformOutput<S, F>>,
2317    I: WorkerInit + Send + Sync + 'static,
2318    I::Error: Send + 'static,
2319    M: StreamMemoryPolicy + Send + 'static,
2320    N: StreamPinPolicy<S, C, F>,
2321{
2322    fn return_credit(&mut self, worker: usize) -> std::result::Result<(), ()> {
2323        self.pool.as_ref().ok_or(())?.pool().return_credit(worker)
2324    }
2325
2326    fn disarm_deadline(&self) {
2327        self.run_context.deadline.disarm();
2328    }
2329
2330    fn set_front_waiter(&mut self, worker: usize, sequence: Option<u64>) {
2331        if let Some(waiters) = M::waiters_mut(&mut self.front_waiters) {
2332            waiters[worker] = sequence;
2333        }
2334    }
2335
2336    fn close(&mut self, poisoned: bool) {
2337        self.exhausted = true;
2338        self.run_context.cancel();
2339        let mut credit_failed = false;
2340        while let Some((_, buffered)) = self.completed.pop() {
2341            credit_failed |= self.return_credit(buffered.worker).is_err();
2342        }
2343        if let Some(mut pool) = self.pool.take() {
2344            let persistent = pool.is_persistent();
2345            let quiesce_failed = persistent && pool.pool_mut().quiesce(self.generation).is_err();
2346            if poisoned || credit_failed || quiesce_failed {
2347                pool.pool_mut().poison();
2348            }
2349        }
2350        self.partial.clear();
2351    }
2352
2353    fn protocol_error(
2354        &mut self,
2355        sequence: Option<u64>,
2356        reason: impl Into<String>,
2357    ) -> StreamLoaderError<S, C, F, I> {
2358        self.disarm_deadline();
2359        self.close(false);
2360        LoaderError::StreamProtocol {
2361            sequence,
2362            reason: reason.into(),
2363        }
2364    }
2365
2366    fn consume(
2367        &mut self,
2368        buffered: BufferedRecord<TransformOutput<S, F>, M::Permit>,
2369    ) -> std::result::Result<bool, StreamLoaderError<S, C, F, I>> {
2370        let BufferedRecord {
2371            worker,
2372            record,
2373            permit,
2374        } = buffered;
2375        let next_sequence = if self.ordered {
2376            let Some(next) = record
2377                .sequence
2378                .expect("ordered records are validated")
2379                .checked_next()
2380            else {
2381                self.run_context.cancel();
2382                if self.return_credit(worker).is_err() {
2383                    self.close(true);
2384                    return Err(LoaderError::ChannelClosed {
2385                        batch: self.next_batch,
2386                    });
2387                }
2388                return Err(self.protocol_error(
2389                    Some(self.next_sequence),
2390                    "stream sequence identifier overflowed",
2391                ));
2392            };
2393            Some(next.get())
2394        } else {
2395            None
2396        };
2397        if self.return_credit(worker).is_err() {
2398            self.close(true);
2399            return Err(LoaderError::ChannelClosed {
2400                batch: self.next_batch,
2401            });
2402        }
2403        if let Some(next_sequence) = next_sequence {
2404            self.next_sequence = next_sequence;
2405        }
2406        drop(permit);
2407        self.partial.push(record.sample);
2408        Ok(self.partial.len() == self.batch_size)
2409    }
2410
2411    fn finish_batch(&mut self) -> std::result::Result<C::Batch, StreamLoaderError<S, C, F, I>> {
2412        let mut replacement = Vec::new();
2413        if replacement.try_reserve_exact(self.batch_size).is_err() {
2414            self.close(false);
2415            return Err(LoaderError::Configuration(invalid_configuration(
2416                "batch_size",
2417                "stream batch allocation is unavailable",
2418            )));
2419        }
2420        let samples = std::mem::replace(&mut self.partial, replacement);
2421        let batch = self.next_batch;
2422        let result = match catch_unwind(AssertUnwindSafe(|| self.collator.collate(samples))) {
2423            Ok(result) => result.map_err(|source| LoaderError::Pipeline {
2424                batch: Some(batch),
2425                worker: None,
2426                source: PipelineError::Collate(source),
2427            }),
2428            Err(_) => Err(LoaderError::CoordinatorPanic {
2429                stage: "stream collation",
2430                batch: Some(batch),
2431            }),
2432        }
2433        .and_then(|value| match self.pin_memory_status {
2434            PinMemoryStatus::Enabled(device) => {
2435                N::pin(value, device).map_err(|source| LoaderError::PinMemory { batch, source })
2436            }
2437            PinMemoryStatus::Disabled | PinMemoryStatus::DisabledNoAccelerator => Ok(value),
2438        });
2439        match result {
2440            Ok(batch) => match self.next_batch.checked_add(1) {
2441                Some(next) => {
2442                    self.next_batch = next;
2443                    self.disarm_deadline();
2444                    Ok(batch)
2445                }
2446                None => {
2447                    self.close(false);
2448                    Err(LoaderError::Configuration(invalid_configuration(
2449                        "batch_size",
2450                        "stream batch sequence overflowed",
2451                    )))
2452                }
2453            },
2454            Err(error) => {
2455                self.disarm_deadline();
2456                self.close(false);
2457                Err(error)
2458            }
2459        }
2460    }
2461
2462    fn map_failure(
2463        &mut self,
2464        worker: usize,
2465        sequence: Option<u64>,
2466        logical_id: Option<u64>,
2467        failure: StreamFailureFor<S, F, I, M>,
2468    ) -> StreamLoaderError<S, C, F, I> {
2469        let batch = Some(self.next_batch);
2470        match failure {
2471            StreamFailure::Source(source) => LoaderError::StreamPipeline {
2472                batch,
2473                worker,
2474                sequence,
2475                logical_id,
2476                source: PipelineError::Source(source),
2477            },
2478            StreamFailure::Transform(source) => LoaderError::StreamPipeline {
2479                batch,
2480                worker,
2481                sequence,
2482                logical_id,
2483                source: PipelineError::Transform(source),
2484            },
2485            StreamFailure::TransformInit(source) => LoaderError::StreamPipeline {
2486                batch: None,
2487                worker,
2488                sequence: None,
2489                logical_id: None,
2490                source: PipelineError::TransformInit(source),
2491            },
2492            StreamFailure::WorkerInit(source) => LoaderError::StreamPipeline {
2493                batch: None,
2494                worker,
2495                sequence: None,
2496                logical_id: None,
2497                source: PipelineError::WorkerInit(source),
2498            },
2499            StreamFailure::Memory(failure) => M::map_failure(failure, worker, sequence, logical_id),
2500            StreamFailure::Panic => LoaderError::StreamWorkerPanic {
2501                worker,
2502                batch,
2503                sequence,
2504                logical_id,
2505            },
2506        }
2507    }
2508}
2509
2510impl<S, C, F, I, M, N> Iterator for StreamLoaderIter<'_, S, C, F, I, M, N>
2511where
2512    S: WorkerSourceFactory,
2513    F: TransformFactory<S::Sample> + Send + Sync + 'static,
2514    F::Transform: Send + 'static,
2515    TransformOutput<S, F>: Send + 'static,
2516    TransformFailure<S, F>: Send + 'static,
2517    F::Error: Send + 'static,
2518    C: Collate<TransformOutput<S, F>>,
2519    I: WorkerInit + Send + Sync + 'static,
2520    I::Error: Send + 'static,
2521    M: StreamMemoryPolicy + Send + 'static,
2522    N: StreamPinPolicy<S, C, F>,
2523{
2524    type Item = std::result::Result<C::Batch, StreamLoaderError<S, C, F, I>>;
2525
2526    fn next(&mut self) -> Option<Self::Item> {
2527        if self.exhausted {
2528            return None;
2529        }
2530        if let Some(error) = self.pending_error.take() {
2531            self.close(false);
2532            return Some(Err(error));
2533        }
2534        if self.source_complete {
2535            self.close(false);
2536            return None;
2537        }
2538        let mut deadline_armed = false;
2539        loop {
2540            if self.ordered
2541                && let Some(index) = self
2542                    .completed
2543                    .iter()
2544                    .position(|(sequence, _)| *sequence == self.next_sequence)
2545            {
2546                let (_, buffered) = self.completed.swap_remove(index);
2547                match self.consume(buffered) {
2548                    Ok(true) => return Some(self.finish_batch()),
2549                    Ok(false) => continue,
2550                    Err(error) => return Some(Err(error)),
2551                }
2552            }
2553
2554            if self.ordered && self.completed.len() == self.reassembly_capacity {
2555                let expected = self.next_sequence;
2556                let found = self
2557                    .completed
2558                    .iter()
2559                    .map(|(sequence, _)| *sequence)
2560                    .min()
2561                    .expect("a full ordered reassembly window is nonempty");
2562                return Some(Err(self.protocol_error(
2563                    Some(expected),
2564                    format!(
2565                        "missing sequence {expected}; bounded reassembly window is full before buffered sequence {found}"
2566                    ),
2567                )));
2568            }
2569
2570            if self.ordered
2571                && self.run_context.byte_budget.is_some()
2572                && let Some(pool) = self.pool.as_ref()
2573                && let Some(front_waiters) = M::waiters(&self.front_waiters)
2574                && front_waiters
2575                    .iter()
2576                    .enumerate()
2577                    .any(|(worker, _)| !pool.pool().terminal.get(worker).copied().unwrap_or(false))
2578                && front_waiters.iter().enumerate().all(|(worker, sequence)| {
2579                    pool.pool().terminal.get(worker).copied().unwrap_or(false)
2580                        || sequence.is_some_and(|sequence| sequence > self.next_sequence)
2581                })
2582            {
2583                let expected = self.next_sequence;
2584                let found = M::waiters(&self.front_waiters)
2585                    .expect("enabled byte accounting has waiter state")
2586                    .iter()
2587                    .flatten()
2588                    .copied()
2589                    .min()
2590                    .expect("a nonterminal byte-budget worker has a front waiter");
2591                return Some(Err(self.protocol_error(
2592                    Some(expected),
2593                    format!(
2594                        "missing sequence {expected}; all active byte-budget workers are waiting at sequence {found} or later"
2595                    ),
2596                )));
2597            }
2598
2599            if self
2600                .pool
2601                .as_ref()
2602                .is_some_and(|pool| pool.pool().all_terminal())
2603            {
2604                if self.ordered
2605                    && let Some(found) = self.completed.iter().map(|(sequence, _)| *sequence).min()
2606                {
2607                    let expected = self.next_sequence;
2608                    return Some(Err(self.protocol_error(
2609                        Some(expected),
2610                        format!("missing sequence {expected} before buffered sequence {found}"),
2611                    )));
2612                }
2613                if self.ordered
2614                    && let Some(expected) = self.expected_records
2615                    && self.next_sequence != expected as u64
2616                {
2617                    let missing = self.next_sequence.min(expected as u64);
2618                    return Some(Err(self.protocol_error(
2619                        Some(missing),
2620                        format!(
2621                            "stream ended after {} ordered records, expected {expected}",
2622                            self.next_sequence
2623                        ),
2624                    )));
2625                }
2626                self.source_complete = true;
2627                if self.partial.is_empty() || self.drop_last {
2628                    self.close(false);
2629                    return None;
2630                }
2631                return Some(self.finish_batch());
2632            }
2633
2634            if !deadline_armed && let Some(timeout) = self.timeout {
2635                self.run_context.deadline.arm(timeout);
2636                deadline_armed = true;
2637            }
2638            let remaining = self
2639                .timeout
2640                .and_then(|_| self.run_context.deadline.remaining());
2641            let received = self
2642                .pool
2643                .as_ref()
2644                .expect("active stream iterator has a pool")
2645                .pool()
2646                .receive(&self.run_context, remaining);
2647            let completion = match received {
2648                StreamReceive::Completion(completion) => completion,
2649                StreamReceive::Timeout => {
2650                    let batch = self.next_batch;
2651                    self.run_context.cancel();
2652                    self.disarm_deadline();
2653                    self.close(false);
2654                    return Some(Err(LoaderError::Timeout { batch }));
2655                }
2656                StreamReceive::Cancelled => {
2657                    self.disarm_deadline();
2658                    self.close(false);
2659                    return Some(Err(LoaderError::Cancelled));
2660                }
2661                StreamReceive::Closed => {
2662                    let batch = self.next_batch;
2663                    self.disarm_deadline();
2664                    self.close(true);
2665                    return Some(Err(LoaderError::ChannelClosed { batch }));
2666                }
2667            };
2668            if completion.generation != self.generation {
2669                if completion.holds_credit && self.return_credit(completion.worker).is_err() {
2670                    self.close(true);
2671                    return Some(Err(LoaderError::ChannelClosed {
2672                        batch: self.next_batch,
2673                    }));
2674                }
2675                continue;
2676            }
2677
2678            match completion.result {
2679                Err(failure) => {
2680                    self.set_front_waiter(completion.worker, None);
2681                    self.run_context.cancel();
2682                    let fatal = matches!(
2683                        &failure,
2684                        StreamFailure::TransformInit(_)
2685                            | StreamFailure::WorkerInit(_)
2686                            | StreamFailure::Panic
2687                    );
2688                    if self
2689                        .pool
2690                        .as_mut()
2691                        .expect("stream pool")
2692                        .pool_mut()
2693                        .account_parts(completion.worker, completion.holds_credit, fatal, fatal)
2694                        .is_err()
2695                    {
2696                        self.close(true);
2697                        return Some(Err(LoaderError::ChannelClosed {
2698                            batch: self.next_batch,
2699                        }));
2700                    }
2701                    let error = self.map_failure(
2702                        completion.worker,
2703                        completion.sequence,
2704                        completion.logical_id,
2705                        failure,
2706                    );
2707                    self.disarm_deadline();
2708                    self.close(fatal);
2709                    return Some(Err(error));
2710                }
2711                Ok(StreamMessage::End) => {
2712                    self.set_front_waiter(completion.worker, None);
2713                    if self
2714                        .pool
2715                        .as_mut()
2716                        .expect("stream pool")
2717                        .pool_mut()
2718                        .account_parts(completion.worker, completion.holds_credit, true, false)
2719                        .is_err()
2720                    {
2721                        self.close(true);
2722                        return Some(Err(LoaderError::ChannelClosed {
2723                            batch: self.next_batch,
2724                        }));
2725                    }
2726                }
2727                Ok(StreamMessage::Waiting(waiting)) => {
2728                    self.set_front_waiter(completion.worker, Some(M::waiting_sequence(waiting)));
2729                }
2730                Ok(StreamMessage::Record(record, permit)) => {
2731                    self.set_front_waiter(completion.worker, None);
2732                    let buffered = BufferedRecord {
2733                        worker: completion.worker,
2734                        record,
2735                        permit,
2736                    };
2737                    if !self.ordered {
2738                        match self.consume(buffered) {
2739                            Ok(true) => return Some(self.finish_batch()),
2740                            Ok(false) => {}
2741                            Err(error) => return Some(Err(error)),
2742                        }
2743                        continue;
2744                    }
2745                    let Some(sequence) = buffered.record.sequence else {
2746                        self.run_context.cancel();
2747                        if self.return_credit(buffered.worker).is_err() {
2748                            self.close(true);
2749                            return Some(Err(LoaderError::ChannelClosed {
2750                                batch: self.next_batch,
2751                            }));
2752                        }
2753                        return Some(Err(self.protocol_error(
2754                            None,
2755                            "ordered stream record did not provide a sequence identifier",
2756                        )));
2757                    };
2758                    let sequence = sequence.get();
2759                    if sequence < self.next_sequence {
2760                        self.run_context.cancel();
2761                        if self.return_credit(buffered.worker).is_err() {
2762                            self.close(true);
2763                            return Some(Err(LoaderError::ChannelClosed {
2764                                batch: self.next_batch,
2765                            }));
2766                        }
2767                        return Some(Err(self.protocol_error(
2768                            Some(sequence),
2769                            format!(
2770                                "duplicate or past sequence {sequence}; next expected is {}",
2771                                self.next_sequence
2772                            ),
2773                        )));
2774                    }
2775                    if self
2776                        .completed
2777                        .iter()
2778                        .any(|(buffered_sequence, _)| *buffered_sequence == sequence)
2779                    {
2780                        self.run_context.cancel();
2781                        if self.return_credit(buffered.worker).is_err() {
2782                            self.close(true);
2783                            return Some(Err(LoaderError::ChannelClosed {
2784                                batch: self.next_batch,
2785                            }));
2786                        }
2787                        return Some(Err(self.protocol_error(
2788                            Some(sequence),
2789                            format!("duplicate buffered sequence {sequence}"),
2790                        )));
2791                    }
2792                    self.completed.push((sequence, buffered));
2793                }
2794            }
2795        }
2796    }
2797}
2798
2799impl<S, C, F, I, M, N> Drop for StreamLoaderIter<'_, S, C, F, I, M, N>
2800where
2801    S: WorkerSourceFactory,
2802    F: TransformFactory<S::Sample>,
2803    C: Collate<TransformOutput<S, F>>,
2804    I: WorkerInit,
2805    M: StreamMemoryPolicy,
2806{
2807    fn drop(&mut self) {
2808        self.run_context.cancel();
2809        let mut failed = false;
2810        while let Some((_, buffered)) = self.completed.pop() {
2811            failed |= self
2812                .pool
2813                .as_ref()
2814                .is_none_or(|pool| pool.pool().return_credit(buffered.worker).is_err());
2815        }
2816        if let Some(mut pool) = self.pool.take()
2817            && pool.is_persistent()
2818            && (failed || pool.pool_mut().quiesce(self.generation).is_err())
2819        {
2820            pool.pool_mut().poison();
2821        }
2822    }
2823}
2824
2825fn invalid_configuration(field: &'static str, reason: impl Into<String>) -> RustTorchError {
2826    RustTorchError::InvalidConfiguration {
2827        field,
2828        reason: reason.into(),
2829    }
2830}
2831
2832#[cfg(test)]
2833mod tests {
2834    use std::convert::Infallible;
2835
2836    use super::*;
2837
2838    type TestCompletion = StreamCompletion<
2839        usize,
2840        (),
2841        Infallible,
2842        Infallible,
2843        Infallible,
2844        Infallible,
2845        Infallible,
2846        Infallible,
2847    >;
2848    type TestResult = StreamResult<
2849        usize,
2850        (),
2851        Infallible,
2852        Infallible,
2853        Infallible,
2854        Infallible,
2855        Infallible,
2856        Infallible,
2857    >;
2858
2859    fn ready_completion(result: TestResult) -> TestCompletion {
2860        StreamCompletion {
2861            worker: 0,
2862            generation: 0,
2863            sequence: None,
2864            logical_id: None,
2865            holds_credit: false,
2866            result,
2867        }
2868    }
2869
2870    fn receive_after_expiry(completion: TestCompletion) -> TestCompletion {
2871        let (sender, receiver) = bounded(1);
2872        sender.send(completion).unwrap();
2873        let context = WorkerRunContext::new(0, 0, 0);
2874        context.deadline.arm(Duration::ZERO);
2875        let shutdown = CancellationToken::new();
2876        match receive_ready_first(&receiver, &context, &shutdown, Some(Duration::ZERO)) {
2877            ReadyFirst::Completion(completion) => completion,
2878            ReadyFirst::Timeout => panic!("ready completion lost to expired deadline"),
2879            ReadyFirst::Cancelled => panic!("ready completion lost to cancellation"),
2880            ReadyFirst::Closed => panic!("ready completion lost to channel close"),
2881        }
2882    }
2883
2884    #[test]
2885    fn queued_error_and_end_beat_an_expired_receive_deadline() {
2886        let error = receive_after_expiry(ready_completion(Err(StreamFailure::Panic)));
2887        assert!(matches!(error.result, Err(StreamFailure::Panic)));
2888
2889        let end = receive_after_expiry(ready_completion(Ok(StreamMessage::End)));
2890        assert!(matches!(end.result, Ok(StreamMessage::End)));
2891    }
2892}