Skip to main content

rusttorch_data/
stream.rs

1//! Explicitly sharded positive-worker stream loading.
2//!
3//! Each worker owns the iterator returned by [`WorkerSourceFactory::create`];
4//! no shared iterator lock is involved. Ordered sources assign one contiguous
5//! global [`SequenceId`] range starting at zero. Unordered sources may omit
6//! sequence IDs, but every [`LogicalSampleId`] must remain stable and unique
7//! within the caller's generation so task randomness has a scheduling-neutral
8//! identity. Records are merged before coordinator batching, so `drop_last`
9//! drops at most one global tail rather than one tail per shard.
10//!
11//! Ordered reassembly retains at most `workers * prefetch_factor` records. If
12//! that entire validated window contains higher IDs while the next ID is
13//! absent, no worker credit remains with which a shard could advance, so the
14//! iterator reports a protocol error instead of waiting for an unreachable end
15//! marker. A source that emits a lower ID after more than this global window of
16//! higher IDs must increase the factor or shard the lower ID onto a worker that
17//! keeps an independent credit available. A worker already producing the
18//! missing low ID holds its credit outside reassembly and is allowed to finish.
19//! The ordered window is one loader-owned flat slot vector reserved and
20//! aggregate-checked before source callbacks, then reused across generations.
21//! Lookup is linear in the deliberately bounded window; increase the factor
22//! only when wider source disorder justifies its memory and scan cost.
23//!
24//! With `prefetch_bytes` enabled, workers measure final post-transform records
25//! and carry cancellation-safe byte permits through the result queue and this
26//! reassembly window. Each ordered shard must then emit strictly increasing
27//! sequence IDs. One bounded front-waiter slot per shard distinguishes a slow
28//! expected record from a globally missing ID without weakening the byte cap.
29//! The coordinator's active item-bounded collation batch is outside the byte
30//! budget. Recursive pinning happens only after successful collation.
31//!
32//! ```
33//! use std::convert::Infallible;
34//! use rusttorch_data::{
35//!     LogicalSampleId, SequenceId, StreamDataLoaderBuilder, VecCollate,
36//!     WorkerContext, WorkerRecord, WorkerSourceFactory,
37//! };
38//!
39//! struct ModuloShards(usize);
40//!
41//! impl WorkerSourceFactory for ModuloShards {
42//!     type Sample = usize;
43//!     type Error = Infallible;
44//!     type Source = std::vec::IntoIter<Result<WorkerRecord<usize>, Infallible>>;
45//!
46//!     fn create(&self, worker: WorkerContext) -> Result<Self::Source, Self::Error> {
47//!         Ok((worker.info.id..self.0)
48//!             .step_by(worker.info.num_workers)
49//!             .map(|value| Ok(WorkerRecord {
50//!                 sequence: Some(SequenceId::new(value as u64)),
51//!                 logical_id: LogicalSampleId::new(value as u64),
52//!                 sample: value,
53//!             }))
54//!             .collect::<Vec<_>>()
55//!             .into_iter())
56//!     }
57//!
58//!     fn exact_len(&self) -> Option<usize> { Some(self.0) }
59//! }
60//!
61//! let mut loader = StreamDataLoaderBuilder::new(ModuloShards(5))
62//!     .workers(2)
63//!     .batch_size(2)
64//!     .collate(VecCollate)
65//!     .build()?;
66//! let batches = loader.iter().collect::<Result<Vec<_>, _>>().unwrap();
67//! assert_eq!(batches, vec![vec![0, 1], vec![2, 3], vec![4]]);
68//! # Ok::<(), rusttorch_core::RustTorchError>(())
69//! ```
70//!
71//! Cancellation and deadlines are cooperative. Iterator and owner drop join
72//! their workers, and therefore wait for a source blocked in non-cooperative
73//! native code until that call returns.
74
75use std::{
76    convert::Infallible,
77    marker::PhantomData,
78    mem::{MaybeUninit, size_of},
79    num::NonZeroUsize,
80    panic::{AssertUnwindSafe, catch_unwind},
81    sync::{Arc, Mutex, atomic::AtomicUsize},
82    thread::{self, JoinHandle},
83    time::Duration,
84};
85
86use crossbeam_channel::{Receiver, RecvError, Sender, TryRecvError, after, bounded, select};
87
88use rusttorch_core::{Device, Result, RustTorchError, available_devices};
89
90use crate::memory::{BudgetError, ByteBudget, MemoryDisabled, MemoryEnabled, MemoryPolicy};
91use crate::worker::WorkerRunContext;
92#[path = "stream_checkpoint.rs"]
93mod exact;
94use crate::{
95    Auto, CancellationToken, CloneTransformFactory, Collate, Deadline, DefaultCollator, Explicit,
96    IdentityTransformFactory, LoaderError, MemoryFootprint, NoWorkerInit, PinDisabled, PinEnabled,
97    PinMemory, PinMemoryStatus, PipelineError, TaskContext, Transform, TransformFactory,
98    WorkerContext, WorkerInfo, WorkerInit, with_worker_info,
99};
100pub use exact::{ExactStreamDataLoader, ExactStreamLoaderIter};
101
102/// A record's position in one ordered global stream generation.
103#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
104pub struct SequenceId(u64);
105
106impl SequenceId {
107    /// Creates an identifier from its checked integer representation.
108    pub const fn new(value: u64) -> Self {
109        Self(value)
110    }
111
112    /// Returns the integer representation.
113    pub const fn get(self) -> u64 {
114        self.0
115    }
116
117    /// Returns the next identifier, or `None` at the integer boundary.
118    pub const fn checked_next(self) -> Option<Self> {
119        match self.0.checked_add(1) {
120            Some(value) => Some(Self(value)),
121            None => None,
122        }
123    }
124}
125
126/// Stable identity used for task randomness and future stream checkpoints.
127#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
128pub struct LogicalSampleId(u64);
129
130impl LogicalSampleId {
131    /// Creates an identifier from its checked integer representation.
132    pub const fn new(value: u64) -> Self {
133        Self(value)
134    }
135
136    /// Returns the integer representation.
137    pub const fn get(self) -> u64 {
138        self.0
139    }
140
141    /// Returns the next identifier, or `None` at the integer boundary.
142    pub const fn checked_next(self) -> Option<Self> {
143        match self.0.checked_add(1) {
144            Some(value) => Some(Self(value)),
145            None => None,
146        }
147    }
148}
149
150/// One typed record produced by an explicitly sharded worker source.
151#[derive(Clone, Debug, Eq, PartialEq)]
152pub struct WorkerRecord<T> {
153    /// Position in the global ordered generation, when supplied.
154    pub sequence: Option<SequenceId>,
155    /// Stable task-randomness and checkpoint identity.
156    pub logical_id: LogicalSampleId,
157    /// Owned source sample.
158    pub sample: T,
159}
160
161/// Creates one independently owned stream shard per worker and generation.
162pub trait WorkerSourceFactory: Send + Sync + 'static {
163    /// Sample produced by every shard.
164    type Sample: Send + 'static;
165    /// Typed source creation or iteration failure.
166    type Error: Send + 'static;
167    /// Independently owned shard iterator.
168    type Source: Iterator<Item = std::result::Result<WorkerRecord<Self::Sample>, Self::Error>>
169        + Send
170        + 'static;
171
172    /// Creates this worker's shard for the supplied generation context.
173    fn create(&self, worker: WorkerContext) -> std::result::Result<Self::Source, Self::Error>;
174
175    /// Returns the exact global record count when known.
176    fn exact_len(&self) -> Option<usize> {
177        None
178    }
179}
180
181type TransformOutput<S, F> =
182    <<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<
183        <S as WorkerSourceFactory>::Sample,
184    >>::Output;
185type TransformFailure<S, F> =
186    <<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<
187        <S as WorkerSourceFactory>::Sample,
188    >>::Error;
189type StreamPipelineError<S, C, F, I> = PipelineError<
190    <S as WorkerSourceFactory>::Error,
191    TransformFailure<S, F>,
192    <C as Collate<TransformOutput<S, F>>>::Error,
193    <F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Error,
194    <I as WorkerInit>::Error,
195>;
196type StreamLoaderError<S, C, F, I> = LoaderError<StreamPipelineError<S, C, F, I>>;
197type ReassemblyEntry<T, P> = (u64, BufferedRecord<T, P>);
198type RetainedReassembly<T, P> = Mutex<Vec<ReassemblyEntry<T, P>>>;
199type StreamFootprint<S, F> = fn(&TransformOutput<S, F>) -> usize;
200
201enum StreamByteFailure {
202    MemoryLimit {
203        limit: usize,
204        actual: usize,
205    },
206    Protocol {
207        sequence: Option<u64>,
208        reason: String,
209    },
210}
211
212trait StreamMemoryPolicy: MemoryPolicy {
213    type Waiters;
214    type Waiting: Send + 'static;
215    type Failure: Send + 'static;
216
217    fn waiter_bytes(workers: usize) -> Result<usize>;
218    fn preflight_waiters(workers: usize) -> Result<()>;
219    fn new_waiters(workers: usize) -> Result<Self::Waiters>;
220    fn waiters(waiters: &Self::Waiters) -> Option<&[Option<u64>]>;
221    fn waiters_mut(waiters: &mut Self::Waiters) -> Option<&mut [Option<u64>]>;
222    fn waiting(sequence: u64) -> Option<Self::Waiting>;
223    fn waiting_sequence(waiting: Self::Waiting) -> u64;
224    fn memory_limit(limit: usize, actual: usize) -> Option<Self::Failure>;
225    fn protocol(sequence: Option<u64>, reason: String) -> Option<Self::Failure>;
226    fn map_failure<E>(
227        failure: Self::Failure,
228        worker: usize,
229        sequence: Option<u64>,
230        logical_id: Option<u64>,
231    ) -> LoaderError<E>;
232}
233
234impl StreamMemoryPolicy for MemoryDisabled {
235    type Waiters = ();
236    type Waiting = Infallible;
237    type Failure = Infallible;
238
239    fn waiter_bytes(_workers: usize) -> Result<usize> {
240        Ok(0)
241    }
242
243    fn preflight_waiters(_workers: usize) -> Result<()> {
244        Ok(())
245    }
246
247    fn new_waiters(_workers: usize) -> Result<Self::Waiters> {
248        Ok(())
249    }
250
251    fn waiters(_waiters: &Self::Waiters) -> Option<&[Option<u64>]> {
252        None
253    }
254
255    fn waiters_mut(_waiters: &mut Self::Waiters) -> Option<&mut [Option<u64>]> {
256        None
257    }
258
259    fn waiting(_sequence: u64) -> Option<Self::Waiting> {
260        None
261    }
262
263    fn waiting_sequence(waiting: Self::Waiting) -> u64 {
264        match waiting {}
265    }
266
267    fn memory_limit(_limit: usize, _actual: usize) -> Option<Self::Failure> {
268        None
269    }
270
271    fn protocol(_sequence: Option<u64>, _reason: String) -> Option<Self::Failure> {
272        None
273    }
274
275    fn map_failure<E>(
276        failure: Self::Failure,
277        _worker: usize,
278        _sequence: Option<u64>,
279        _logical_id: Option<u64>,
280    ) -> LoaderError<E> {
281        match failure {}
282    }
283}
284
285impl StreamMemoryPolicy for MemoryEnabled {
286    type Waiters = Vec<Option<u64>>;
287    type Waiting = u64;
288    type Failure = StreamByteFailure;
289
290    fn waiter_bytes(workers: usize) -> Result<usize> {
291        workers
292            .checked_mul(size_of::<Option<u64>>())
293            .ok_or_else(|| capacity_error("stream byte waiter storage exceeds usize"))
294    }
295
296    fn preflight_waiters(workers: usize) -> Result<()> {
297        preflight_vec::<Option<u64>>(workers, "stream byte waiter state")
298    }
299
300    fn new_waiters(workers: usize) -> Result<Self::Waiters> {
301        let mut waiters = Vec::new();
302        reserve_exact(&mut waiters, workers, "stream byte waiter state")?;
303        waiters.resize(workers, None);
304        Ok(waiters)
305    }
306
307    fn waiters(waiters: &Self::Waiters) -> Option<&[Option<u64>]> {
308        Some(waiters)
309    }
310
311    fn waiters_mut(waiters: &mut Self::Waiters) -> Option<&mut [Option<u64>]> {
312        Some(waiters)
313    }
314
315    fn waiting(sequence: u64) -> Option<Self::Waiting> {
316        Some(sequence)
317    }
318
319    fn waiting_sequence(waiting: Self::Waiting) -> u64 {
320        waiting
321    }
322
323    fn memory_limit(limit: usize, actual: usize) -> Option<Self::Failure> {
324        Some(StreamByteFailure::MemoryLimit { limit, actual })
325    }
326
327    fn protocol(sequence: Option<u64>, reason: String) -> Option<Self::Failure> {
328        Some(StreamByteFailure::Protocol { sequence, reason })
329    }
330
331    fn map_failure<E>(
332        failure: Self::Failure,
333        worker: usize,
334        sequence: Option<u64>,
335        logical_id: Option<u64>,
336    ) -> LoaderError<E> {
337        match failure {
338            StreamByteFailure::MemoryLimit { limit, actual } => LoaderError::MemoryLimit {
339                batch: None,
340                worker: Some(worker),
341                sequence,
342                logical_id,
343                limit,
344                actual,
345            },
346            StreamByteFailure::Protocol { sequence, reason } => {
347                LoaderError::StreamProtocol { sequence, reason }
348            }
349        }
350    }
351}
352
353#[doc(hidden)]
354pub trait StreamPinPolicy<S, C, F>
355where
356    S: WorkerSourceFactory,
357    F: TransformFactory<S::Sample>,
358    C: Collate<TransformOutput<S, F>>,
359{
360    fn pin(batch: C::Batch, device: Device) -> Result<C::Batch>;
361}
362
363impl<S, C, F> StreamPinPolicy<S, C, F> for PinDisabled
364where
365    S: WorkerSourceFactory,
366    F: TransformFactory<S::Sample>,
367    C: Collate<TransformOutput<S, F>>,
368{
369    fn pin(batch: C::Batch, _device: Device) -> Result<C::Batch> {
370        Ok(batch)
371    }
372}
373
374impl<S, C, F, Q> StreamPinPolicy<S, C, F> for PinEnabled<Q>
375where
376    S: WorkerSourceFactory,
377    F: TransformFactory<S::Sample>,
378    C: Collate<TransformOutput<S, F>>,
379    C::Batch: PinMemory,
380{
381    fn pin(batch: C::Batch, device: Device) -> Result<C::Batch> {
382        batch.pin_memory(device)
383    }
384}
385
386type StreamCompletionFor<S, F, I, M> = StreamCompletion<
387    TransformOutput<S, F>,
388    <M as MemoryPolicy>::Permit,
389    <M as StreamMemoryPolicy>::Waiting,
390    <M as StreamMemoryPolicy>::Failure,
391    <S as WorkerSourceFactory>::Error,
392    TransformFailure<S, F>,
393    <F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Error,
394    <I as WorkerInit>::Error,
395>;
396type StreamFailureFor<S, F, I, M> = StreamFailure<
397    <S as WorkerSourceFactory>::Error,
398    TransformFailure<S, F>,
399    <F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Error,
400    <I as WorkerInit>::Error,
401    <M as StreamMemoryPolicy>::Failure,
402>;
403type StreamResult<T, P, W, ME, SE, TE, FE, IE> =
404    std::result::Result<StreamMessage<T, P, W>, StreamFailure<SE, TE, FE, IE, ME>>;
405type StreamCompletionSender<T, P, W, ME, SE, TE, FE, IE> =
406    Sender<StreamCompletion<T, P, W, ME, SE, TE, FE, IE>>;
407
408enum StreamFailure<SE, TE, FE, IE, ME> {
409    Source(SE),
410    Transform(TE),
411    TransformInit(FE),
412    WorkerInit(IE),
413    Memory(ME),
414    Panic,
415}
416
417enum StreamMessage<T, P, W> {
418    Record(WorkerRecord<T>, P),
419    Waiting(W),
420    End,
421}
422
423struct StreamCompletion<T, P, W, ME, SE, TE, FE, IE> {
424    worker: usize,
425    generation: u64,
426    sequence: Option<u64>,
427    logical_id: Option<u64>,
428    holds_credit: bool,
429    result: StreamResult<T, P, W, ME, SE, TE, FE, IE>,
430}
431
432enum StreamControl {
433    Begin(WorkerRunContext),
434}
435
436enum StreamReceive<S, F, I, M>
437where
438    S: WorkerSourceFactory,
439    F: TransformFactory<S::Sample>,
440    I: WorkerInit,
441    M: StreamMemoryPolicy,
442{
443    Completion(StreamCompletionFor<S, F, I, M>),
444    Timeout,
445    Cancelled,
446    Closed,
447}
448
449enum ReadyFirst<T> {
450    Completion(T),
451    Timeout,
452    Cancelled,
453    Closed,
454}
455
456const MAX_STREAM_QUEUE_ALLOCATION_BYTES: usize = 64 * 1024 * 1024;
457const CHANNEL_CONTROL_BLOCK_ALLOWANCE_BYTES: usize = 2 * 1024;
458
459#[allow(dead_code)]
460#[repr(C)]
461struct CapacitySlot<T> {
462    stamp: AtomicUsize,
463    message: MaybeUninit<T>,
464}
465
466fn validate_stream_capacity<S, F, I, M>(
467    workers: usize,
468    prefetch_factor: usize,
469    outstanding: usize,
470    batch_size: usize,
471    reassembly_slots: usize,
472) -> Result<()>
473where
474    S: WorkerSourceFactory,
475    F: TransformFactory<S::Sample>,
476    I: WorkerInit,
477    M: StreamMemoryPolicy,
478{
479    validate_stream_capacity_extra::<S, F, I, M>(
480        workers,
481        prefetch_factor,
482        outstanding,
483        batch_size,
484        reassembly_slots,
485        0,
486    )
487}
488
489fn validate_stream_capacity_extra<S, F, I, M>(
490    workers: usize,
491    prefetch_factor: usize,
492    outstanding: usize,
493    batch_size: usize,
494    reassembly_slots: usize,
495    extra: usize,
496) -> Result<()>
497where
498    S: WorkerSourceFactory,
499    F: TransformFactory<S::Sample>,
500    I: WorkerInit,
501    M: StreamMemoryPolicy,
502{
503    let expected = workers
504        .checked_mul(prefetch_factor)
505        .ok_or_else(|| capacity_error("workers multiplied by prefetch_factor exceeds usize"))?;
506    if expected != outstanding {
507        return Err(capacity_error(
508            "outstanding capacity does not match workers multiplied by prefetch_factor",
509        ));
510    }
511    for capacity in [1, prefetch_factor, outstanding] {
512        capacity
513            .checked_add(1)
514            .and_then(usize::checked_next_power_of_two)
515            .and_then(|mark_bit| mark_bit.checked_mul(2))
516            .ok_or_else(|| capacity_error("bounded channel ring arithmetic overflowed"))?;
517    }
518
519    let credit_bytes = outstanding
520        .checked_mul(size_of::<CapacitySlot<()>>())
521        .ok_or_else(|| capacity_error("stream credit storage exceeds usize"))?;
522    let result_bytes = outstanding
523        .checked_mul(size_of::<CapacitySlot<StreamCompletionFor<S, F, I, M>>>())
524        .ok_or_else(|| capacity_error("stream result storage exceeds usize"))?;
525    let control_bytes = workers
526        .checked_mul(size_of::<CapacitySlot<StreamControl>>())
527        .ok_or_else(|| capacity_error("stream control storage exceeds usize"))?;
528    let batch_bytes = batch_size
529        .checked_mul(size_of::<TransformOutput<S, F>>())
530        .and_then(|bytes| bytes.checked_mul(2))
531        .ok_or_else(|| capacity_error("stream batch storage exceeds usize"))?;
532    let reassembly_bytes = reassembly_slots
533        .checked_mul(size_of::<
534            ReassemblyEntry<TransformOutput<S, F>, <M as MemoryPolicy>::Permit>,
535        >())
536        .ok_or_else(|| capacity_error("ordered stream reassembly exceeds usize"))?;
537    let per_worker = size_of::<WorkerInfo>()
538        .checked_add(size_of::<(Sender<()>, Receiver<()>)>())
539        .and_then(|bytes| {
540            bytes.checked_add(size_of::<(Sender<StreamControl>, Receiver<StreamControl>)>())
541        })
542        .and_then(|bytes| bytes.checked_add(size_of::<Sender<()>>()))
543        .and_then(|bytes| bytes.checked_add(size_of::<Sender<StreamControl>>()))
544        .and_then(|bytes| bytes.checked_add(size_of::<JoinHandle<()>>()))
545        .and_then(|bytes| bytes.checked_add(size_of::<bool>()))
546        .ok_or_else(|| capacity_error("stream worker bookkeeping exceeds usize"))?;
547    let waiter_bytes = M::waiter_bytes(workers)?;
548    let bookkeeping = workers
549        .checked_mul(per_worker)
550        .and_then(|bytes| bytes.checked_add(waiter_bytes))
551        .ok_or_else(|| capacity_error("stream worker bookkeeping exceeds usize"))?;
552    let channel_count = workers
553        .checked_mul(2)
554        .and_then(|count| count.checked_add(1))
555        .ok_or_else(|| capacity_error("stream channel count exceeds usize"))?;
556    let channel_bytes = channel_count
557        .checked_mul(CHANNEL_CONTROL_BLOCK_ALLOWANCE_BYTES)
558        .ok_or_else(|| capacity_error("stream channel controls exceed usize"))?;
559    let aggregate = credit_bytes
560        .checked_add(result_bytes)
561        .and_then(|bytes| bytes.checked_add(control_bytes))
562        .and_then(|bytes| bytes.checked_add(batch_bytes))
563        .and_then(|bytes| bytes.checked_add(reassembly_bytes))
564        .and_then(|bytes| bytes.checked_add(bookkeeping))
565        .and_then(|bytes| bytes.checked_add(channel_bytes))
566        .and_then(|bytes| bytes.checked_add(extra))
567        .ok_or_else(|| capacity_error("aggregate stream queue storage exceeds usize"))?;
568    if aggregate > MAX_STREAM_QUEUE_ALLOCATION_BYTES {
569        return Err(capacity_error(format!(
570            "aggregate stream queue storage requires {aggregate} bytes, above the {MAX_STREAM_QUEUE_ALLOCATION_BYTES}-byte safety ceiling"
571        )));
572    }
573    preflight_channel_storage::<()>(outstanding)?;
574    preflight_channel_storage::<StreamControl>(workers)?;
575    preflight_channel_storage::<StreamCompletionFor<S, F, I, M>>(outstanding)?;
576    preflight_vec::<TransformOutput<S, F>>(batch_size, "stream batch")?;
577    M::preflight_waiters(workers)?;
578    Ok(())
579}
580
581fn preflight_channel_storage<T>(capacity: usize) -> Result<()> {
582    let mut reservation: Vec<MaybeUninit<CapacitySlot<T>>> = Vec::new();
583    reservation
584        .try_reserve_exact(capacity)
585        .map_err(|error| capacity_error(format!("bounded stream storage is unavailable: {error}")))
586}
587
588fn preflight_vec<T>(capacity: usize, label: &'static str) -> Result<()> {
589    let mut reservation: Vec<T> = Vec::new();
590    reservation
591        .try_reserve_exact(capacity)
592        .map_err(|error| capacity_error(format!("{label} capacity is unavailable: {error}")))
593}
594
595fn capacity_error(reason: impl Into<String>) -> RustTorchError {
596    invalid_configuration("prefetch_factor", reason)
597}
598
599struct StreamWorkerPool<S, F, I, M>
600where
601    S: WorkerSourceFactory,
602    F: TransformFactory<S::Sample>,
603    I: WorkerInit,
604    M: StreamMemoryPolicy,
605{
606    controls: Vec<Sender<StreamControl>>,
607    credits: Vec<Sender<()>>,
608    results: Option<Receiver<StreamCompletionFor<S, F, I, M>>>,
609    handles: Vec<JoinHandle<()>>,
610    shutdown: CancellationToken,
611    active: Option<WorkerRunContext>,
612    terminal: Vec<bool>,
613    terminal_count: usize,
614    fatal_exit: bool,
615    poisoned: bool,
616}
617
618impl<S, F, I, M> StreamWorkerPool<S, F, I, M>
619where
620    S: WorkerSourceFactory,
621    F: TransformFactory<S::Sample> + Send + Sync + 'static,
622    F::Transform: Send + 'static,
623    TransformOutput<S, F>: Send + 'static,
624    TransformFailure<S, F>: Send + 'static,
625    F::Error: Send + 'static,
626    I: WorkerInit + Send + Sync + 'static,
627    I::Error: Send + 'static,
628    M: StreamMemoryPolicy + Send + 'static,
629{
630    #[allow(clippy::too_many_arguments)]
631    fn new(
632        source_factory: Arc<S>,
633        transform_factory: Arc<F>,
634        initializer: Arc<I>,
635        configuration: StreamConfiguration,
636        generation: u64,
637        outstanding: usize,
638        reassembly_slots: usize,
639        footprint: Option<StreamFootprint<S, F>>,
640    ) -> Result<Self> {
641        validate_stream_capacity::<S, F, I, M>(
642            configuration.workers,
643            configuration.prefetch_factor,
644            outstanding,
645            configuration.batch_size,
646            reassembly_slots,
647        )?;
648
649        let (result_sender, result_receiver) =
650            bounded_checked(outstanding, "stream result channel")?;
651        let mut credit_lanes = Vec::new();
652        let mut control_lanes = Vec::new();
653        reserve_exact(
654            &mut credit_lanes,
655            configuration.workers,
656            "stream credit channels",
657        )?;
658        reserve_exact(
659            &mut control_lanes,
660            configuration.workers,
661            "stream control channels",
662        )?;
663        for _ in 0..configuration.workers {
664            let (credit_sender, credit_receiver) =
665                bounded_checked(configuration.prefetch_factor, "stream credit channel")?;
666            for _ in 0..configuration.prefetch_factor {
667                credit_sender.try_send(()).map_err(|_| {
668                    capacity_error("validated stream credit channel rejected a token")
669                })?;
670            }
671            credit_lanes.push((credit_sender, credit_receiver));
672            control_lanes.push(bounded_checked(1, "stream control channel")?);
673        }
674
675        let shutdown = CancellationToken::new();
676        let mut pool = Self {
677            controls: Vec::new(),
678            credits: Vec::new(),
679            results: Some(result_receiver),
680            handles: Vec::new(),
681            shutdown: shutdown.clone(),
682            active: None,
683            terminal: Vec::new(),
684            terminal_count: 0,
685            fatal_exit: false,
686            poisoned: false,
687        };
688        reserve_exact(&mut pool.controls, configuration.workers, "stream controls")?;
689        reserve_exact(&mut pool.credits, configuration.workers, "stream credits")?;
690        reserve_exact(&mut pool.handles, configuration.workers, "stream workers")?;
691        reserve_exact(
692            &mut pool.terminal,
693            configuration.workers,
694            "stream terminal state",
695        )?;
696        pool.terminal.resize(configuration.workers, false);
697
698        for (id, ((credit_sender, credit_receiver), (control_sender, control_receiver))) in
699            credit_lanes.into_iter().zip(control_lanes).enumerate()
700        {
701            let info = WorkerInfo::from_loader_seed(
702                id,
703                configuration.workers,
704                configuration.loader_seed,
705                configuration.rank,
706                generation,
707            )?;
708            pool.credits.push(credit_sender);
709            pool.controls.push(control_sender);
710            let source_factory = Arc::clone(&source_factory);
711            let transform_factory = Arc::clone(&transform_factory);
712            let initializer = Arc::clone(&initializer);
713            let results = result_sender.clone();
714            let shutdown = shutdown.clone();
715            let handle = thread::Builder::new()
716                .name(format!("rusttorch-stream-worker-{id}"))
717                .spawn(move || {
718                    run_stream_worker::<S, F, I, M>(
719                        info,
720                        generation,
721                        source_factory,
722                        transform_factory,
723                        initializer,
724                        control_receiver,
725                        credit_receiver,
726                        results,
727                        shutdown,
728                        footprint,
729                        configuration.ordered,
730                    );
731                })
732                .map_err(|error| RustTorchError::BackendUnavailable {
733                    backend: "data loader stream worker threads",
734                    reason: error.to_string(),
735                })?;
736            pool.handles.push(handle);
737        }
738        drop(result_sender);
739        Ok(pool)
740    }
741}
742
743impl<S, F, I, M> StreamWorkerPool<S, F, I, M>
744where
745    S: WorkerSourceFactory,
746    F: TransformFactory<S::Sample>,
747    I: WorkerInit,
748    M: StreamMemoryPolicy,
749{
750    fn start_generation(&mut self, context: WorkerRunContext) -> std::result::Result<(), ()> {
751        if self.poisoned || self.active.is_some() || self.shutdown.is_cancelled() {
752            return Err(());
753        }
754        self.terminal.fill(false);
755        self.terminal_count = 0;
756        self.fatal_exit = false;
757        self.active = Some(context.clone());
758        for control in &self.controls {
759            if control.send(StreamControl::Begin(context.clone())).is_err() {
760                self.poisoned = true;
761                self.shutdown();
762                return Err(());
763            }
764        }
765        Ok(())
766    }
767
768    fn receive(
769        &self,
770        context: &WorkerRunContext,
771        timeout: Option<Duration>,
772    ) -> StreamReceive<S, F, I, M> {
773        let results = self
774            .results
775            .as_ref()
776            .expect("stream result receiver exists before shutdown");
777        match receive_ready_first(results, context, &self.shutdown, timeout) {
778            ReadyFirst::Completion(completion) => StreamReceive::Completion(completion),
779            ReadyFirst::Timeout => StreamReceive::Timeout,
780            ReadyFirst::Cancelled => StreamReceive::Cancelled,
781            ReadyFirst::Closed => StreamReceive::Closed,
782        }
783    }
784
785    fn account(
786        &mut self,
787        completion: &StreamCompletionFor<S, F, I, M>,
788    ) -> std::result::Result<(), ()> {
789        let fatal = matches!(
790            &completion.result,
791            Err(StreamFailure::TransformInit(_)
792                | StreamFailure::WorkerInit(_)
793                | StreamFailure::Panic)
794        );
795        let terminal = fatal || matches!(&completion.result, Ok(StreamMessage::End));
796        self.account_parts(completion.worker, completion.holds_credit, terminal, fatal)
797    }
798
799    fn account_parts(
800        &mut self,
801        worker: usize,
802        holds_credit: bool,
803        terminal: bool,
804        fatal: bool,
805    ) -> std::result::Result<(), ()> {
806        if holds_credit {
807            self.return_credit(worker)?;
808        }
809        if fatal {
810            self.fatal_exit = true;
811        }
812        if terminal {
813            let Some(seen) = self.terminal.get_mut(worker) else {
814                return Err(());
815            };
816            if !*seen {
817                *seen = true;
818                self.terminal_count += 1;
819            }
820        }
821        Ok(())
822    }
823
824    fn return_credit(&self, worker: usize) -> std::result::Result<(), ()> {
825        let Some(credit) = self.credits.get(worker) else {
826            return Err(());
827        };
828        credit.try_send(()).map_err(|_| ())
829    }
830
831    fn all_terminal(&self) -> bool {
832        self.terminal_count == self.terminal.len()
833    }
834
835    fn quiesce(&mut self, generation: u64) -> std::result::Result<(), ()> {
836        let Some(active) = self.active.take() else {
837            return Ok(());
838        };
839        active.cancel();
840        while !self.all_terminal() {
841            let completion = self.results.as_ref().ok_or(())?.recv().map_err(|_| ())?;
842            if completion.generation == generation {
843                self.account(&completion)?;
844            } else if completion.holds_credit {
845                self.return_credit(completion.worker)?;
846            }
847        }
848        while let Some(results) = self.results.as_ref() {
849            match results.try_recv() {
850                Ok(completion) => self.account(&completion)?,
851                Err(TryRecvError::Empty) => break,
852                Err(TryRecvError::Disconnected) => return Err(()),
853            }
854        }
855        if self.fatal_exit {
856            self.poisoned = true;
857            self.shutdown();
858            return Err(());
859        }
860        Ok(())
861    }
862
863    fn poison(&mut self) {
864        self.poisoned = true;
865        self.shutdown();
866    }
867
868    fn shutdown(&mut self) {
869        if let Some(active) = self.active.take() {
870            active.cancel();
871        }
872        self.shutdown.cancel();
873        self.controls.clear();
874        self.credits.clear();
875        self.results.take();
876        for handle in self.handles.drain(..) {
877            let _ = handle.join();
878        }
879    }
880}
881
882impl<S, F, I, M> Drop for StreamWorkerPool<S, F, I, M>
883where
884    S: WorkerSourceFactory,
885    F: TransformFactory<S::Sample>,
886    I: WorkerInit,
887    M: StreamMemoryPolicy,
888{
889    fn drop(&mut self) {
890        self.shutdown();
891    }
892}
893
894fn receive_ready_first<T>(
895    results: &Receiver<T>,
896    context: &WorkerRunContext,
897    shutdown: &CancellationToken,
898    timeout: Option<Duration>,
899) -> ReadyFirst<T> {
900    match results.try_recv() {
901        Ok(completion) => return ReadyFirst::Completion(completion),
902        Err(TryRecvError::Disconnected) => return ReadyFirst::Closed,
903        Err(TryRecvError::Empty) => {}
904    }
905    if let Some(timeout) = timeout {
906        let timer = after(timeout);
907        select! {
908            recv(results) -> result => map_ready_receive(result),
909            recv(context.cancellation.signal()) -> _ => match results.try_recv() {
910                Ok(completion) => ReadyFirst::Completion(completion),
911                Err(TryRecvError::Empty) => ReadyFirst::Cancelled,
912                Err(TryRecvError::Disconnected) => ReadyFirst::Closed,
913            },
914            recv(shutdown.signal()) -> _ => ReadyFirst::Closed,
915            recv(timer) -> _ => match results.try_recv() {
916                Ok(completion) => ReadyFirst::Completion(completion),
917                Err(TryRecvError::Empty) => ReadyFirst::Timeout,
918                Err(TryRecvError::Disconnected) => ReadyFirst::Closed,
919            },
920        }
921    } else {
922        select! {
923            recv(results) -> result => map_ready_receive(result),
924            recv(context.cancellation.signal()) -> _ => match results.try_recv() {
925                Ok(completion) => ReadyFirst::Completion(completion),
926                Err(TryRecvError::Empty) => ReadyFirst::Cancelled,
927                Err(TryRecvError::Disconnected) => ReadyFirst::Closed,
928            },
929            recv(shutdown.signal()) -> _ => ReadyFirst::Closed,
930        }
931    }
932}
933
934fn map_ready_receive<T>(result: std::result::Result<T, RecvError>) -> ReadyFirst<T> {
935    match result {
936        Ok(completion) => ReadyFirst::Completion(completion),
937        Err(_) => ReadyFirst::Closed,
938    }
939}
940
941fn reserve_exact<T>(values: &mut Vec<T>, capacity: usize, label: &'static str) -> Result<()> {
942    values
943        .try_reserve_exact(capacity)
944        .map_err(|error| capacity_error(format!("{label} capacity is unavailable: {error}")))
945}
946
947fn bounded_checked<T>(capacity: usize, label: &'static str) -> Result<(Sender<T>, Receiver<T>)> {
948    catch_unwind(AssertUnwindSafe(|| bounded(capacity))).map_err(|_| {
949        capacity_error(format!(
950            "{label} rejected the validated capacity {capacity}"
951        ))
952    })
953}
954
955#[allow(clippy::too_many_arguments)]
956fn run_stream_worker<S, F, I, M>(
957    info: WorkerInfo,
958    initialization_generation: u64,
959    source_factory: Arc<S>,
960    transform_factory: Arc<F>,
961    initializer: Arc<I>,
962    controls: Receiver<StreamControl>,
963    credits: Receiver<()>,
964    results: Sender<StreamCompletionFor<S, F, I, M>>,
965    shutdown: CancellationToken,
966    footprint: Option<StreamFootprint<S, F>>,
967    ordered: bool,
968) where
969    S: WorkerSourceFactory,
970    F: TransformFactory<S::Sample> + Send + Sync + 'static,
971    F::Transform: Send + 'static,
972    TransformOutput<S, F>: Send + 'static,
973    TransformFailure<S, F>: Send + 'static,
974    F::Error: Send + 'static,
975    I: WorkerInit + Send + Sync + 'static,
976    I::Error: Send + 'static,
977    M: StreamMemoryPolicy + Send + 'static,
978{
979    let lifecycle = WorkerContext::new(info, shutdown.clone(), shutdown.paired_deadline());
980    let mut active: Option<(WorkerRunContext, Option<u64>, Option<u64>, bool)> = None;
981    let outcome = catch_unwind(AssertUnwindSafe(|| {
982        with_worker_info(info, || {
983            if let Err(error) = initializer.initialize(&lifecycle) {
984                send_stream_failure(
985                    &results,
986                    &shutdown,
987                    info.id,
988                    initialization_generation,
989                    None,
990                    None,
991                    false,
992                    StreamFailure::WorkerInit(error),
993                );
994                return true;
995            }
996            let mut transform = match transform_factory.create(Some(&lifecycle)) {
997                Ok(transform) => transform,
998                Err(error) => {
999                    send_stream_failure(
1000                        &results,
1001                        &shutdown,
1002                        info.id,
1003                        initialization_generation,
1004                        None,
1005                        None,
1006                        false,
1007                        StreamFailure::TransformInit(error),
1008                    );
1009                    return true;
1010                }
1011            };
1012
1013            loop {
1014                let run = select! {
1015                    recv(controls) -> control => match control {
1016                        Ok(StreamControl::Begin(run)) => run,
1017                        Err(_) => return false,
1018                    },
1019                    recv(shutdown.signal()) -> _ => return false,
1020                };
1021                active = Some((run.clone(), None, None, false));
1022                run_stream_generation::<S, F, I, M>(
1023                    info,
1024                    &source_factory,
1025                    &mut transform,
1026                    &credits,
1027                    &results,
1028                    &shutdown,
1029                    &run,
1030                    &mut active,
1031                    footprint,
1032                    ordered,
1033                );
1034                active = None;
1035                if shutdown.is_cancelled() {
1036                    return false;
1037                }
1038            }
1039        })
1040    }));
1041    let fatal = match outcome {
1042        Ok(fatal) => fatal,
1043        Err(_) => {
1044            let (generation, sequence, logical_id, holds_credit) = active
1045                .as_ref()
1046                .map(|(run, sequence, logical_id, held)| {
1047                    (run.generation, *sequence, *logical_id, *held)
1048                })
1049                .unwrap_or((initialization_generation, None, None, false));
1050            send_stream_failure(
1051                &results,
1052                &shutdown,
1053                info.id,
1054                generation,
1055                sequence,
1056                logical_id,
1057                holds_credit,
1058                StreamFailure::Panic,
1059            );
1060            true
1061        }
1062    };
1063    if fatal {
1064        shutdown.wait_cancelled();
1065    }
1066}
1067
1068#[allow(clippy::too_many_arguments)]
1069fn run_stream_generation<S, F, I, M>(
1070    info: WorkerInfo,
1071    source_factory: &S,
1072    transform: &mut F::Transform,
1073    credits: &Receiver<()>,
1074    results: &Sender<StreamCompletionFor<S, F, I, M>>,
1075    shutdown: &CancellationToken,
1076    run: &WorkerRunContext,
1077    active: &mut Option<(WorkerRunContext, Option<u64>, Option<u64>, bool)>,
1078    footprint: Option<StreamFootprint<S, F>>,
1079    ordered: bool,
1080) where
1081    S: WorkerSourceFactory,
1082    F: TransformFactory<S::Sample> + Send + Sync + 'static,
1083    F::Transform: Send + 'static,
1084    TransformOutput<S, F>: Send + 'static,
1085    TransformFailure<S, F>: Send + 'static,
1086    F::Error: Send + 'static,
1087    I: WorkerInit + Send + Sync + 'static,
1088    I::Error: Send + 'static,
1089    M: StreamMemoryPolicy,
1090{
1091    let generation_info = WorkerInfo::from_loader_seed(
1092        info.id,
1093        info.num_workers,
1094        run.loader_seed,
1095        info.rank,
1096        run.generation,
1097    )
1098    .expect("validated stream worker identity remains valid for every generation");
1099    let context = WorkerContext::new(
1100        generation_info,
1101        run.cancellation.clone(),
1102        run.deadline.clone(),
1103    );
1104    let mut source = match source_factory.create(context.clone()) {
1105        Ok(source) if context.check().is_ok() => source,
1106        Ok(source) => {
1107            drop(source);
1108            send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, false);
1109            return;
1110        }
1111        Err(error) => {
1112            send_stream_failure(
1113                results,
1114                shutdown,
1115                info.id,
1116                run.generation,
1117                None,
1118                None,
1119                false,
1120                StreamFailure::Source(error),
1121            );
1122            run.cancel();
1123            send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, false);
1124            return;
1125        }
1126    };
1127
1128    let mut last_sequence = None;
1129    loop {
1130        let acquired = select! {
1131            recv(credits) -> credit => credit.is_ok(),
1132            recv(run.cancellation.signal()) -> _ => false,
1133            recv(shutdown.signal()) -> _ => return,
1134        };
1135        if !acquired {
1136            drop(source);
1137            send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, false);
1138            return;
1139        }
1140        active.as_mut().expect("active stream generation").3 = true;
1141        if context.check().is_err() {
1142            drop(source);
1143            send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1144            active.as_mut().expect("active stream generation").3 = false;
1145            return;
1146        }
1147
1148        let record = match source.next() {
1149            Some(Ok(record)) => {
1150                if context.check().is_err() {
1151                    drop(source);
1152                    send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1153                    active.as_mut().expect("active stream generation").3 = false;
1154                    return;
1155                }
1156                record
1157            }
1158            None => {
1159                drop(source);
1160                send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1161                active.as_mut().expect("active stream generation").3 = false;
1162                return;
1163            }
1164            Some(Err(error)) => {
1165                drop(source);
1166                send_stream_failure(
1167                    results,
1168                    shutdown,
1169                    info.id,
1170                    run.generation,
1171                    None,
1172                    None,
1173                    true,
1174                    StreamFailure::Source(error),
1175                );
1176                active.as_mut().expect("active stream generation").3 = false;
1177                run.cancel();
1178                send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, false);
1179                return;
1180            }
1181        };
1182        let sequence = record.sequence.map(SequenceId::get);
1183        let logical_id = record.logical_id.get();
1184        {
1185            let active = active.as_mut().expect("active stream generation");
1186            active.1 = sequence;
1187            active.2 = Some(logical_id);
1188        }
1189        let task_context = TaskContext {
1190            loader_seed: run.loader_seed,
1191            epoch: run.epoch,
1192            rank: info.rank,
1193            logical_sample: logical_id,
1194            stage: 0,
1195            cancellation: run.cancellation.clone(),
1196            deadline: run.deadline.clone(),
1197        };
1198        let sample = match transform.transform(record.sample, &task_context) {
1199            Ok(sample) if context.check().is_ok() => sample,
1200            Ok(_) => {
1201                drop(source);
1202                send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1203                active.as_mut().expect("active stream generation").3 = false;
1204                return;
1205            }
1206            Err(error) => {
1207                drop(source);
1208                send_stream_failure(
1209                    results,
1210                    shutdown,
1211                    info.id,
1212                    run.generation,
1213                    sequence,
1214                    Some(logical_id),
1215                    true,
1216                    StreamFailure::Transform(error),
1217                );
1218                active.as_mut().expect("active stream generation").3 = false;
1219                run.cancel();
1220                send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, false);
1221                return;
1222            }
1223        };
1224        let permit = if let Some(footprint) = footprint {
1225            let actual = footprint(&sample);
1226            let budget = run
1227                .byte_budget
1228                .as_ref()
1229                .expect("enabled stream byte accounting has a generation budget");
1230            let acquired = if ordered {
1231                let Some(sequence) = sequence else {
1232                    drop(source);
1233                    send_stream_failure(
1234                        results,
1235                        shutdown,
1236                        info.id,
1237                        run.generation,
1238                        None,
1239                        Some(logical_id),
1240                        true,
1241                        StreamFailure::Memory(
1242                            M::protocol(
1243                                None,
1244                                "ordered byte-bounded stream record did not provide a sequence identifier".to_owned(),
1245                            )
1246                            .expect("enabled stream accounting has protocol failure storage"),
1247                        ),
1248                    );
1249                    active.as_mut().expect("active stream generation").3 = false;
1250                    run.cancel();
1251                    send_stream_end::<S, F, I, M>(
1252                        results,
1253                        shutdown,
1254                        info.id,
1255                        run.generation,
1256                        false,
1257                    );
1258                    return;
1259                };
1260                if last_sequence.is_some_and(|previous| sequence <= previous) {
1261                    drop(source);
1262                    send_stream_failure(
1263                        results,
1264                        shutdown,
1265                        info.id,
1266                        run.generation,
1267                        Some(sequence),
1268                        Some(logical_id),
1269                        true,
1270                        StreamFailure::Memory(
1271                            M::protocol(
1272                                Some(sequence),
1273                                format!(
1274                                    "worker {} emitted non-increasing sequence {sequence}",
1275                                    info.id
1276                                ),
1277                            )
1278                            .expect("enabled stream accounting has protocol failure storage"),
1279                        ),
1280                    );
1281                    active.as_mut().expect("active stream generation").3 = false;
1282                    run.cancel();
1283                    send_stream_end::<S, F, I, M>(
1284                        results,
1285                        shutdown,
1286                        info.id,
1287                        run.generation,
1288                        false,
1289                    );
1290                    return;
1291                }
1292                last_sequence = Some(sequence);
1293                if !send_stream_completion(
1294                    results,
1295                    shutdown,
1296                    Some(&run.cancellation),
1297                    StreamCompletion {
1298                        worker: info.id,
1299                        generation: run.generation,
1300                        sequence: Some(sequence),
1301                        logical_id: Some(logical_id),
1302                        holds_credit: false,
1303                        result: Ok(StreamMessage::Waiting(
1304                            M::waiting(sequence)
1305                                .expect("enabled stream accounting has waiter messages"),
1306                        )),
1307                    },
1308                ) {
1309                    drop(source);
1310                    send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1311                    active.as_mut().expect("active stream generation").3 = false;
1312                    return;
1313                }
1314                budget.acquire_ordered(sequence, actual)
1315            } else {
1316                budget.acquire(actual)
1317            };
1318            match acquired {
1319                Ok(permit) => Some(permit),
1320                Err(BudgetError::Cancelled) => {
1321                    drop(source);
1322                    send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1323                    active.as_mut().expect("active stream generation").3 = false;
1324                    return;
1325                }
1326                Err(BudgetError::Oversize { limit, actual }) => {
1327                    drop(source);
1328                    send_stream_failure(
1329                        results,
1330                        shutdown,
1331                        info.id,
1332                        run.generation,
1333                        sequence,
1334                        Some(logical_id),
1335                        true,
1336                        StreamFailure::Memory(
1337                            M::memory_limit(limit, actual)
1338                                .expect("enabled stream accounting has memory-limit storage"),
1339                        ),
1340                    );
1341                    active.as_mut().expect("active stream generation").3 = false;
1342                    run.cancel();
1343                    send_stream_end::<S, F, I, M>(
1344                        results,
1345                        shutdown,
1346                        info.id,
1347                        run.generation,
1348                        false,
1349                    );
1350                    return;
1351                }
1352                Err(BudgetError::SequenceAlreadyAdmitted) => {
1353                    drop(source);
1354                    send_stream_failure(
1355                        results,
1356                        shutdown,
1357                        info.id,
1358                        run.generation,
1359                        sequence,
1360                        Some(logical_id),
1361                        true,
1362                        StreamFailure::Memory(
1363                            M::protocol(
1364                                sequence,
1365                                "sequence was already admitted to the byte budget".to_owned(),
1366                            )
1367                            .expect("enabled stream accounting has protocol failure storage"),
1368                        ),
1369                    );
1370                    active.as_mut().expect("active stream generation").3 = false;
1371                    run.cancel();
1372                    send_stream_end::<S, F, I, M>(
1373                        results,
1374                        shutdown,
1375                        info.id,
1376                        run.generation,
1377                        false,
1378                    );
1379                    return;
1380                }
1381            }
1382        } else {
1383            None
1384        };
1385        let sent = send_stream_completion(
1386            results,
1387            shutdown,
1388            Some(&run.cancellation),
1389            StreamCompletion {
1390                worker: info.id,
1391                generation: run.generation,
1392                sequence,
1393                logical_id: Some(logical_id),
1394                holds_credit: true,
1395                result: Ok(StreamMessage::Record(
1396                    WorkerRecord {
1397                        sequence: record.sequence,
1398                        logical_id: record.logical_id,
1399                        sample,
1400                    },
1401                    M::permit(permit),
1402                )),
1403            },
1404        );
1405        if sent {
1406            let active = active.as_mut().expect("active stream generation");
1407            active.1 = None;
1408            active.2 = None;
1409            active.3 = false;
1410        } else {
1411            drop(source);
1412            send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1413            active.as_mut().expect("active stream generation").3 = false;
1414            return;
1415        }
1416    }
1417}
1418
1419fn send_stream_end<S, F, I, M>(
1420    results: &Sender<StreamCompletionFor<S, F, I, M>>,
1421    shutdown: &CancellationToken,
1422    worker: usize,
1423    generation: u64,
1424    holds_credit: bool,
1425) where
1426    S: WorkerSourceFactory,
1427    F: TransformFactory<S::Sample>,
1428    I: WorkerInit,
1429    M: StreamMemoryPolicy,
1430{
1431    let _ = send_stream_completion(
1432        results,
1433        shutdown,
1434        None,
1435        StreamCompletion {
1436            worker,
1437            generation,
1438            sequence: None,
1439            logical_id: None,
1440            holds_credit,
1441            result: Ok(StreamMessage::End),
1442        },
1443    );
1444}
1445
1446#[allow(clippy::too_many_arguments)]
1447fn send_stream_failure<T, P, W, ME, SE, TE, FE, IE>(
1448    results: &StreamCompletionSender<T, P, W, ME, SE, TE, FE, IE>,
1449    shutdown: &CancellationToken,
1450    worker: usize,
1451    generation: u64,
1452    sequence: Option<u64>,
1453    logical_id: Option<u64>,
1454    holds_credit: bool,
1455    failure: StreamFailure<SE, TE, FE, IE, ME>,
1456) {
1457    let _ = send_stream_completion(
1458        results,
1459        shutdown,
1460        None,
1461        StreamCompletion {
1462            worker,
1463            generation,
1464            sequence,
1465            logical_id,
1466            holds_credit,
1467            result: Err(failure),
1468        },
1469    );
1470}
1471
1472fn send_stream_completion<T, P, W, ME, SE, TE, FE, IE>(
1473    results: &StreamCompletionSender<T, P, W, ME, SE, TE, FE, IE>,
1474    shutdown: &CancellationToken,
1475    cancellation: Option<&CancellationToken>,
1476    completion: StreamCompletion<T, P, W, ME, SE, TE, FE, IE>,
1477) -> bool {
1478    if let Some(cancellation) = cancellation {
1479        select! {
1480            send(results, completion) -> result => result.is_ok(),
1481            recv(cancellation.signal()) -> _ => false,
1482            recv(shutdown.signal()) -> _ => false,
1483        }
1484    } else {
1485        select! {
1486            send(results, completion) -> result => result.is_ok(),
1487            recv(shutdown.signal()) -> _ => false,
1488        }
1489    }
1490}
1491
1492#[derive(Clone, Copy)]
1493struct StreamConfiguration {
1494    workers: usize,
1495    batch_size: usize,
1496    drop_last: bool,
1497    prefetch_factor: usize,
1498    ordered: bool,
1499    persistent_workers: bool,
1500    timeout: Option<Duration>,
1501    loader_seed: u64,
1502    epoch: u64,
1503    rank: usize,
1504    prefetch_bytes: Option<NonZeroUsize>,
1505    pin_memory: StreamPinRequest,
1506}
1507
1508#[derive(Clone, Copy)]
1509enum StreamPinRequest {
1510    Disabled,
1511    Auto,
1512    Explicit(Device),
1513}
1514
1515impl Default for StreamConfiguration {
1516    fn default() -> Self {
1517        Self {
1518            workers: 1,
1519            batch_size: 1,
1520            drop_last: false,
1521            prefetch_factor: 2,
1522            ordered: true,
1523            persistent_workers: false,
1524            timeout: None,
1525            loader_seed: 0,
1526            epoch: 0,
1527            rank: 0,
1528            prefetch_bytes: None,
1529            pin_memory: StreamPinRequest::Disabled,
1530        }
1531    }
1532}
1533
1534/// Builder for an explicitly sharded positive-worker stream loader.
1535pub struct StreamDataLoaderBuilder<
1536    S,
1537    C = DefaultCollator,
1538    F = IdentityTransformFactory,
1539    I = NoWorkerInit,
1540    M = MemoryDisabled,
1541    N = PinDisabled,
1542    K = crate::CheckpointDisabled,
1543> {
1544    factory: S,
1545    collator: C,
1546    transform_factory: F,
1547    worker_init: I,
1548    configuration: StreamConfiguration,
1549    checkpoint: K,
1550    states: PhantomData<(M, N)>,
1551}
1552
1553impl<S> StreamDataLoaderBuilder<S>
1554where
1555    S: WorkerSourceFactory,
1556{
1557    /// Starts a builder with one worker, ordered delivery, batch size one,
1558    /// prefetch factor two, no timeout, and no persistence.
1559    pub fn new(factory: S) -> Self {
1560        Self {
1561            factory,
1562            collator: DefaultCollator,
1563            transform_factory: IdentityTransformFactory,
1564            worker_init: NoWorkerInit,
1565            configuration: StreamConfiguration::default(),
1566            checkpoint: crate::CheckpointDisabled,
1567            states: PhantomData,
1568        }
1569    }
1570}
1571
1572impl<S, C, F, I, M, N, K> StreamDataLoaderBuilder<S, C, F, I, M, N, K> {
1573    /// Sets the positive stream worker count.
1574    pub fn workers(mut self, workers: usize) -> Self {
1575        self.configuration.workers = workers;
1576        self
1577    }
1578
1579    /// Sets the global coordinator batch size.
1580    pub fn batch_size(mut self, batch_size: usize) -> Self {
1581        self.configuration.batch_size = batch_size;
1582        self
1583    }
1584
1585    /// Selects whether the one final global short batch is omitted.
1586    pub fn drop_last(mut self, drop_last: bool) -> Self {
1587        self.configuration.drop_last = drop_last;
1588        self
1589    }
1590
1591    /// Sets bounded unpublished records permitted per worker.
1592    ///
1593    /// In ordered mode, filling the complete global reassembly window with
1594    /// higher IDs before the next ID is observable is a stream protocol error.
1595    pub fn prefetch_factor(mut self, factor: usize) -> Self {
1596        self.configuration.prefetch_factor = factor;
1597        self
1598    }
1599
1600    /// Enables a nonzero byte budget for final post-transform records.
1601    ///
1602    /// Ordered byte-bounded shards must emit strictly increasing sequence IDs.
1603    /// The coordinator tracks one bounded front waiter per shard and reports a
1604    /// protocol error when every active shard has advanced beyond a missing ID.
1605    pub fn prefetch_bytes(
1606        mut self,
1607        limit: NonZeroUsize,
1608    ) -> StreamDataLoaderBuilder<S, C, F, I, MemoryEnabled, N, K> {
1609        self.configuration.prefetch_bytes = Some(limit);
1610        StreamDataLoaderBuilder {
1611            factory: self.factory,
1612            collator: self.collator,
1613            transform_factory: self.transform_factory,
1614            worker_init: self.worker_init,
1615            configuration: self.configuration,
1616            checkpoint: self.checkpoint,
1617            states: PhantomData,
1618        }
1619    }
1620
1621    /// Enables automatic recursive pinning for CUDA device zero when available.
1622    pub fn pin_memory(mut self) -> StreamDataLoaderBuilder<S, C, F, I, M, PinEnabled<Auto>, K> {
1623        self.configuration.pin_memory = StreamPinRequest::Auto;
1624        StreamDataLoaderBuilder {
1625            factory: self.factory,
1626            collator: self.collator,
1627            transform_factory: self.transform_factory,
1628            worker_init: self.worker_init,
1629            configuration: self.configuration,
1630            checkpoint: self.checkpoint,
1631            states: PhantomData,
1632        }
1633    }
1634
1635    /// Enables recursive pinning for one explicit, available CUDA device.
1636    pub fn pin_memory_for(
1637        mut self,
1638        device: Device,
1639    ) -> StreamDataLoaderBuilder<S, C, F, I, M, PinEnabled<Explicit>, K> {
1640        self.configuration.pin_memory = StreamPinRequest::Explicit(device);
1641        StreamDataLoaderBuilder {
1642            factory: self.factory,
1643            collator: self.collator,
1644            transform_factory: self.transform_factory,
1645            worker_init: self.worker_init,
1646            configuration: self.configuration,
1647            checkpoint: self.checkpoint,
1648            states: PhantomData,
1649        }
1650    }
1651
1652    /// Selects globally sequenced or completion-order delivery.
1653    pub fn ordered(mut self, ordered: bool) -> Self {
1654        self.configuration.ordered = ordered;
1655        self
1656    }
1657
1658    /// Selects globally sequenced or completion-order delivery.
1659    pub fn in_order(self, in_order: bool) -> Self {
1660        self.ordered(in_order)
1661    }
1662
1663    /// Sets the deterministic loader seed.
1664    pub fn seed(mut self, seed: u64) -> Self {
1665        self.configuration.loader_seed = seed;
1666        self
1667    }
1668
1669    /// Sets the initial stream epoch.
1670    pub fn epoch(mut self, epoch: u64) -> Self {
1671        self.configuration.epoch = epoch;
1672        self
1673    }
1674
1675    /// Sets the distributed rank included in worker and task contexts.
1676    pub fn rank(mut self, rank: usize) -> Self {
1677        self.configuration.rank = rank;
1678        self
1679    }
1680
1681    /// Sets the cooperative timeout for each blocking iterator call.
1682    pub fn timeout(mut self, timeout: Duration) -> Self {
1683        self.configuration.timeout = (!timeout.is_zero()).then_some(timeout);
1684        self
1685    }
1686
1687    /// Selects loader-owned persistent worker threads.
1688    pub fn persistent_workers(mut self, persistent: bool) -> Self {
1689        self.configuration.persistent_workers = persistent;
1690        self
1691    }
1692
1693    /// Replaces coordinator collation.
1694    pub fn collate<C2>(self, collator: C2) -> StreamDataLoaderBuilder<S, C2, F, I, M, N, K> {
1695        StreamDataLoaderBuilder {
1696            factory: self.factory,
1697            collator,
1698            transform_factory: self.transform_factory,
1699            worker_init: self.worker_init,
1700            configuration: self.configuration,
1701            checkpoint: self.checkpoint,
1702            states: PhantomData,
1703        }
1704    }
1705
1706    /// Clones one transform template for each worker lifecycle.
1707    pub fn transform<T>(
1708        self,
1709        transform: T,
1710    ) -> StreamDataLoaderBuilder<S, C, CloneTransformFactory<T>, I, M, N, K> {
1711        StreamDataLoaderBuilder {
1712            factory: self.factory,
1713            collator: self.collator,
1714            transform_factory: CloneTransformFactory::new(transform),
1715            worker_init: self.worker_init,
1716            configuration: self.configuration,
1717            checkpoint: self.checkpoint,
1718            states: PhantomData,
1719        }
1720    }
1721
1722    /// Replaces worker transform construction.
1723    pub fn transform_factory<F2>(
1724        self,
1725        factory: F2,
1726    ) -> StreamDataLoaderBuilder<S, C, F2, I, M, N, K> {
1727        StreamDataLoaderBuilder {
1728            factory: self.factory,
1729            collator: self.collator,
1730            transform_factory: factory,
1731            worker_init: self.worker_init,
1732            configuration: self.configuration,
1733            checkpoint: self.checkpoint,
1734            states: PhantomData,
1735        }
1736    }
1737
1738    /// Replaces worker initialization.
1739    pub fn worker_init<I2>(self, worker_init: I2) -> StreamDataLoaderBuilder<S, C, F, I2, M, N, K> {
1740        StreamDataLoaderBuilder {
1741            factory: self.factory,
1742            collator: self.collator,
1743            transform_factory: self.transform_factory,
1744            worker_init,
1745            configuration: self.configuration,
1746            checkpoint: self.checkpoint,
1747            states: PhantomData,
1748        }
1749    }
1750
1751    /// Validates configuration and constructs the stream owner.
1752    fn build_inner(
1753        self,
1754        footprint: Option<StreamFootprint<S, F>>,
1755    ) -> Result<StreamDataLoader<S, C, F, I, M, N>>
1756    where
1757        S: WorkerSourceFactory,
1758        F: TransformFactory<S::Sample>,
1759        C: Collate<TransformOutput<S, F>>,
1760        I: WorkerInit,
1761        M: StreamMemoryPolicy,
1762    {
1763        let pin_memory_status = resolve_stream_pin(self.configuration.pin_memory)?;
1764        if self.configuration.workers == 0 {
1765            return Err(invalid_configuration(
1766                "workers",
1767                "stream workers must be positive; use batches or batches_with_collate for an ordinary iterator",
1768            ));
1769        }
1770        if self.configuration.batch_size == 0 {
1771            return Err(invalid_configuration(
1772                "batch_size",
1773                "must be greater than zero",
1774            ));
1775        }
1776        if self.configuration.prefetch_factor == 0 {
1777            return Err(invalid_configuration(
1778                "prefetch_factor",
1779                "must be greater than zero",
1780            ));
1781        }
1782        if self
1783            .configuration
1784            .timeout
1785            .is_some_and(|timeout| !Deadline::can_represent(timeout))
1786        {
1787            return Err(invalid_configuration(
1788                "timeout",
1789                "exceeds the platform monotonic clock range",
1790            ));
1791        }
1792        let outstanding_capacity = self
1793            .configuration
1794            .workers
1795            .checked_mul(self.configuration.prefetch_factor)
1796            .ok_or_else(|| {
1797                invalid_configuration(
1798                    "prefetch_factor",
1799                    "workers multiplied by prefetch_factor exceeds usize",
1800                )
1801            })?;
1802        validate_stream_capacity::<S, F, I, M>(
1803            self.configuration.workers,
1804            self.configuration.prefetch_factor,
1805            outstanding_capacity,
1806            self.configuration.batch_size,
1807            usize::from(self.configuration.ordered) * outstanding_capacity,
1808        )?;
1809        let mut ordered_reassembly = Vec::new();
1810        if self.configuration.ordered {
1811            ordered_reassembly
1812                .try_reserve_exact(outstanding_capacity)
1813                .map_err(|error| {
1814                    capacity_error(format!(
1815                        "ordered stream reassembly capacity is unavailable: {error}"
1816                    ))
1817                })?;
1818        }
1819        validate_stream_capacity::<S, F, I, M>(
1820            self.configuration.workers,
1821            self.configuration.prefetch_factor,
1822            outstanding_capacity,
1823            self.configuration.batch_size,
1824            ordered_reassembly.capacity(),
1825        )?;
1826        let exact_len = self.factory.exact_len();
1827        Ok(StreamDataLoader {
1828            factory: Arc::new(self.factory),
1829            collator: self.collator,
1830            transform_factory: Arc::new(self.transform_factory),
1831            worker_init: Arc::new(self.worker_init),
1832            configuration: self.configuration,
1833            outstanding_capacity,
1834            exact_len,
1835            next_generation: 0,
1836            persistent_pool: None,
1837            ordered_reassembly: Mutex::new(ordered_reassembly),
1838            footprint,
1839            pin_memory_status,
1840            policies: PhantomData,
1841        })
1842    }
1843}
1844
1845fn stream_resident_bytes<T: MemoryFootprint>(value: &T) -> usize {
1846    value.resident_bytes()
1847}
1848
1849fn resolve_stream_pin(request: StreamPinRequest) -> Result<PinMemoryStatus> {
1850    match request {
1851        StreamPinRequest::Disabled => Ok(PinMemoryStatus::Disabled),
1852        StreamPinRequest::Auto => {
1853            let capabilities = available_devices();
1854            if capabilities.cuda && capabilities.cuda_device_count > 0 {
1855                Ok(PinMemoryStatus::Enabled(Device::Cuda(0)))
1856            } else {
1857                Ok(PinMemoryStatus::DisabledNoAccelerator)
1858            }
1859        }
1860        StreamPinRequest::Explicit(Device::Cuda(index)) => {
1861            let capabilities = available_devices();
1862            if capabilities.cuda && index < capabilities.cuda_device_count {
1863                Ok(PinMemoryStatus::Enabled(Device::Cuda(index)))
1864            } else {
1865                Err(invalid_configuration(
1866                    "pin_memory",
1867                    format!(
1868                        "CUDA device {index} was requested, but the linked runtime exposes {} available CUDA device(s)",
1869                        capabilities.cuda_device_count
1870                    ),
1871                ))
1872            }
1873        }
1874        StreamPinRequest::Explicit(device) => Err(invalid_configuration(
1875            "pin_memory",
1876            format!("only an available CUDA device can back pinned host memory, got {device:?}"),
1877        )),
1878    }
1879}
1880
1881impl<S, C, F, I> StreamDataLoaderBuilder<S, C, F, I, MemoryDisabled, PinDisabled> {
1882    /// Validates configuration and builds an item-bounded, unpinned stream.
1883    pub fn build(self) -> Result<StreamDataLoader<S, C, F, I, MemoryDisabled, PinDisabled>>
1884    where
1885        S: WorkerSourceFactory,
1886        F: TransformFactory<S::Sample>,
1887        C: Collate<TransformOutput<S, F>>,
1888        I: WorkerInit,
1889    {
1890        self.build_inner(None)
1891    }
1892}
1893
1894impl<S, C, F, I> StreamDataLoaderBuilder<S, C, F, I, MemoryEnabled, PinDisabled> {
1895    /// Validates configuration and builds a byte-bounded, unpinned stream.
1896    pub fn build(self) -> Result<StreamDataLoader<S, C, F, I, MemoryEnabled, PinDisabled>>
1897    where
1898        S: WorkerSourceFactory,
1899        F: TransformFactory<S::Sample>,
1900        TransformOutput<S, F>: MemoryFootprint,
1901        C: Collate<TransformOutput<S, F>>,
1902        I: WorkerInit,
1903    {
1904        self.build_inner(Some(stream_resident_bytes::<TransformOutput<S, F>>))
1905    }
1906}
1907
1908impl<S, C, F, I, Q> StreamDataLoaderBuilder<S, C, F, I, MemoryDisabled, PinEnabled<Q>> {
1909    /// Validates configuration and builds an item-bounded pinned stream.
1910    pub fn build(self) -> Result<StreamDataLoader<S, C, F, I, MemoryDisabled, PinEnabled<Q>>>
1911    where
1912        S: WorkerSourceFactory,
1913        F: TransformFactory<S::Sample>,
1914        C: Collate<TransformOutput<S, F>>,
1915        C::Batch: PinMemory,
1916        I: WorkerInit,
1917    {
1918        self.build_inner(None)
1919    }
1920}
1921
1922impl<S, C, F, I, Q> StreamDataLoaderBuilder<S, C, F, I, MemoryEnabled, PinEnabled<Q>> {
1923    /// Validates configuration and builds a byte-bounded pinned stream.
1924    pub fn build(self) -> Result<StreamDataLoader<S, C, F, I, MemoryEnabled, PinEnabled<Q>>>
1925    where
1926        S: WorkerSourceFactory,
1927        F: TransformFactory<S::Sample>,
1928        TransformOutput<S, F>: MemoryFootprint,
1929        C: Collate<TransformOutput<S, F>>,
1930        C::Batch: PinMemory,
1931        I: WorkerInit,
1932    {
1933        self.build_inner(Some(stream_resident_bytes::<TransformOutput<S, F>>))
1934    }
1935}
1936
1937/// Owned, re-iterable explicitly sharded stream loader.
1938#[allow(private_bounds)]
1939pub struct StreamDataLoader<
1940    S,
1941    C,
1942    F = IdentityTransformFactory,
1943    I = NoWorkerInit,
1944    M = MemoryDisabled,
1945    N = PinDisabled,
1946> where
1947    S: WorkerSourceFactory,
1948    F: TransformFactory<S::Sample>,
1949    C: Collate<TransformOutput<S, F>>,
1950    I: WorkerInit,
1951    M: StreamMemoryPolicy,
1952{
1953    factory: Arc<S>,
1954    collator: C,
1955    transform_factory: Arc<F>,
1956    worker_init: Arc<I>,
1957    configuration: StreamConfiguration,
1958    outstanding_capacity: usize,
1959    exact_len: Option<usize>,
1960    next_generation: u64,
1961    persistent_pool: Option<StreamWorkerPool<S, F, I, M>>,
1962    // `iter` has exclusive loader access; this wrapper preserves `Sync` for
1963    // output that is `Send` but not `Sync` without runtime contention.
1964    ordered_reassembly: RetainedReassembly<TransformOutput<S, F>, M::Permit>,
1965    footprint: Option<StreamFootprint<S, F>>,
1966    pin_memory_status: PinMemoryStatus,
1967    policies: PhantomData<(M, N)>,
1968}
1969
1970#[allow(private_bounds)]
1971impl<S, C, F, I, M, N> StreamDataLoader<S, C, F, I, M, N>
1972where
1973    S: WorkerSourceFactory,
1974    F: TransformFactory<S::Sample>,
1975    C: Collate<TransformOutput<S, F>>,
1976    I: WorkerInit,
1977    M: StreamMemoryPolicy,
1978{
1979    /// Returns the exact global batch count when the factory is sized.
1980    pub fn len(&self) -> Option<usize> {
1981        self.exact_len.map(|length| {
1982            let complete = length / self.configuration.batch_size;
1983            complete
1984                + usize::from(
1985                    !self.configuration.drop_last
1986                        && !length.is_multiple_of(self.configuration.batch_size),
1987                )
1988        })
1989    }
1990
1991    /// Returns whether the sized stream yields no batches.
1992    pub fn is_empty(&self) -> Option<bool> {
1993        self.len().map(|length| length == 0)
1994    }
1995
1996    /// Returns the configured worker count.
1997    pub fn workers(&self) -> usize {
1998        self.configuration.workers
1999    }
2000
2001    /// Returns the effective records-prefetched-per-worker factor.
2002    pub fn effective_prefetch_factor(&self) -> NonZeroUsize {
2003        NonZeroUsize::new(self.configuration.prefetch_factor)
2004            .expect("validated prefetch factor is nonzero")
2005    }
2006
2007    /// Returns the effective post-transform record byte budget.
2008    pub fn effective_prefetch_bytes(&self) -> Option<NonZeroUsize> {
2009        self.configuration.prefetch_bytes
2010    }
2011
2012    /// Returns whether recursive batch pinning was requested.
2013    pub fn pin_memory_enabled(&self) -> bool {
2014        !matches!(self.pin_memory_status, PinMemoryStatus::Disabled)
2015    }
2016
2017    /// Returns the effective recursive pinning behavior.
2018    pub fn pin_memory_status(&self) -> PinMemoryStatus {
2019        self.pin_memory_status
2020    }
2021
2022    /// Returns whether ordered delivery is enabled.
2023    pub fn is_ordered(&self) -> bool {
2024        self.configuration.ordered
2025    }
2026
2027    /// Returns the configured epoch.
2028    pub fn epoch(&self) -> u64 {
2029        self.configuration.epoch
2030    }
2031
2032    /// Selects the epoch used by future iterator generations.
2033    pub fn set_epoch(&mut self, epoch: u64) {
2034        self.configuration.epoch = epoch;
2035    }
2036}
2037
2038struct BufferedRecord<T, P> {
2039    worker: usize,
2040    record: WorkerRecord<T>,
2041    permit: P,
2042}
2043
2044enum StreamIteratorPool<'a, S, F, I, M>
2045where
2046    S: WorkerSourceFactory,
2047    F: TransformFactory<S::Sample>,
2048    I: WorkerInit,
2049    M: StreamMemoryPolicy,
2050{
2051    Owned(StreamWorkerPool<S, F, I, M>),
2052    Persistent(&'a mut StreamWorkerPool<S, F, I, M>),
2053}
2054
2055impl<S, F, I, M> StreamIteratorPool<'_, S, F, I, M>
2056where
2057    S: WorkerSourceFactory,
2058    F: TransformFactory<S::Sample>,
2059    I: WorkerInit,
2060    M: StreamMemoryPolicy,
2061{
2062    fn pool(&self) -> &StreamWorkerPool<S, F, I, M> {
2063        match self {
2064            Self::Owned(pool) => pool,
2065            Self::Persistent(pool) => pool,
2066        }
2067    }
2068
2069    fn pool_mut(&mut self) -> &mut StreamWorkerPool<S, F, I, M> {
2070        match self {
2071            Self::Owned(pool) => pool,
2072            Self::Persistent(pool) => pool,
2073        }
2074    }
2075
2076    fn is_persistent(&self) -> bool {
2077        matches!(self, Self::Persistent(_))
2078    }
2079}
2080
2081#[allow(private_bounds)]
2082impl<S, C, F, I, M, N> StreamDataLoader<S, C, F, I, M, N>
2083where
2084    S: WorkerSourceFactory,
2085    F: TransformFactory<S::Sample> + Send + Sync + 'static,
2086    F::Transform: Send + 'static,
2087    TransformOutput<S, F>: Send + 'static,
2088    TransformFailure<S, F>: Send + 'static,
2089    F::Error: Send + 'static,
2090    C: Collate<TransformOutput<S, F>>,
2091    I: WorkerInit + Send + Sync + 'static,
2092    I::Error: Send + 'static,
2093    M: StreamMemoryPolicy + Send + 'static,
2094    N: StreamPinPolicy<S, C, F>,
2095{
2096    /// Starts a fresh explicitly sharded worker generation.
2097    pub fn iter(&mut self) -> StreamLoaderIter<'_, S, C, F, I, M, N> {
2098        let ordered_reassembly = match self.ordered_reassembly.get_mut() {
2099            Ok(reassembly) => reassembly,
2100            Err(poisoned) => poisoned.into_inner(),
2101        };
2102        ordered_reassembly.clear();
2103        let reassembly_allocation_slots = ordered_reassembly.capacity();
2104        let generation = self.next_generation;
2105        let run_context = WorkerRunContext::new(
2106            generation,
2107            self.configuration.loader_seed,
2108            self.configuration.epoch,
2109        )
2110        .with_byte_budget(
2111            self.configuration
2112                .prefetch_bytes
2113                .map(|limit| ByteBudget::new(limit.get())),
2114        );
2115        let mut pending_error = None;
2116        if let Some(next) = generation.checked_add(1) {
2117            self.next_generation = next;
2118        } else {
2119            pending_error = Some(LoaderError::Configuration(invalid_configuration(
2120                "workers",
2121                "stream iterator generation overflowed",
2122            )));
2123        }
2124
2125        let mut partial = Vec::new();
2126        if partial
2127            .try_reserve_exact(self.configuration.batch_size)
2128            .is_err()
2129        {
2130            pending_error = Some(LoaderError::Configuration(invalid_configuration(
2131                "batch_size",
2132                "stream batch allocation is unavailable",
2133            )));
2134        }
2135
2136        let front_waiters = match M::new_waiters(self.configuration.workers) {
2137            Ok(waiters) => waiters,
2138            Err(error) => {
2139                pending_error = Some(LoaderError::Configuration(error));
2140                M::new_waiters(0).expect("zero waiter state is always allocatable")
2141            }
2142        };
2143
2144        let mut pool = None;
2145        if pending_error.is_none() {
2146            if self.configuration.persistent_workers {
2147                if self.persistent_pool.is_none() {
2148                    match StreamWorkerPool::new(
2149                        Arc::clone(&self.factory),
2150                        Arc::clone(&self.transform_factory),
2151                        Arc::clone(&self.worker_init),
2152                        self.configuration,
2153                        generation,
2154                        self.outstanding_capacity,
2155                        reassembly_allocation_slots,
2156                        self.footprint,
2157                    ) {
2158                        Ok(created) => self.persistent_pool = Some(created),
2159                        Err(error) => pending_error = Some(LoaderError::Configuration(error)),
2160                    }
2161                }
2162                if pending_error.is_none() {
2163                    let persistent = self
2164                        .persistent_pool
2165                        .as_mut()
2166                        .expect("persistent stream pool was created");
2167                    if persistent.start_generation(run_context.clone()).is_err() {
2168                        pending_error = Some(LoaderError::ChannelClosed { batch: 0 });
2169                    } else {
2170                        pool = Some(StreamIteratorPool::Persistent(persistent));
2171                    }
2172                }
2173            } else {
2174                match StreamWorkerPool::new(
2175                    Arc::clone(&self.factory),
2176                    Arc::clone(&self.transform_factory),
2177                    Arc::clone(&self.worker_init),
2178                    self.configuration,
2179                    generation,
2180                    self.outstanding_capacity,
2181                    reassembly_allocation_slots,
2182                    self.footprint,
2183                ) {
2184                    Ok(mut created) => {
2185                        if created.start_generation(run_context.clone()).is_err() {
2186                            pending_error = Some(LoaderError::ChannelClosed { batch: 0 });
2187                        } else {
2188                            pool = Some(StreamIteratorPool::Owned(created));
2189                        }
2190                    }
2191                    Err(error) => pending_error = Some(LoaderError::Configuration(error)),
2192                }
2193            }
2194        }
2195
2196        StreamLoaderIter {
2197            collator: &mut self.collator,
2198            pool,
2199            run_context,
2200            generation,
2201            ordered: self.configuration.ordered,
2202            timeout: self.configuration.timeout,
2203            batch_size: self.configuration.batch_size,
2204            drop_last: self.configuration.drop_last,
2205            expected_records: self.exact_len,
2206            reassembly_capacity: self.outstanding_capacity,
2207            completed: ordered_reassembly,
2208            partial,
2209            next_sequence: 0,
2210            next_batch: 0,
2211            pending_error,
2212            source_complete: false,
2213            exhausted: false,
2214            front_waiters,
2215            pin_memory_status: self.pin_memory_status,
2216            marker: PhantomData,
2217        }
2218    }
2219}
2220
2221/// One borrowing iterator generation from a [`StreamDataLoader`].
2222#[allow(private_bounds)]
2223pub struct StreamLoaderIter<'a, S, C, F, I, M = MemoryDisabled, N = PinDisabled>
2224where
2225    S: WorkerSourceFactory,
2226    F: TransformFactory<S::Sample>,
2227    C: Collate<TransformOutput<S, F>>,
2228    I: WorkerInit,
2229    M: StreamMemoryPolicy,
2230{
2231    collator: &'a mut C,
2232    pool: Option<StreamIteratorPool<'a, S, F, I, M>>,
2233    run_context: WorkerRunContext,
2234    generation: u64,
2235    ordered: bool,
2236    timeout: Option<Duration>,
2237    batch_size: usize,
2238    drop_last: bool,
2239    expected_records: Option<usize>,
2240    reassembly_capacity: usize,
2241    completed: &'a mut Vec<ReassemblyEntry<TransformOutput<S, F>, M::Permit>>,
2242    partial: Vec<TransformOutput<S, F>>,
2243    next_sequence: u64,
2244    next_batch: u64,
2245    pending_error: Option<StreamLoaderError<S, C, F, I>>,
2246    source_complete: bool,
2247    exhausted: bool,
2248    front_waiters: M::Waiters,
2249    pin_memory_status: PinMemoryStatus,
2250    marker: PhantomData<&'a mut (S, C, F, I, M, N)>,
2251}
2252
2253#[allow(private_bounds)]
2254impl<S, C, F, I, M, N> StreamLoaderIter<'_, S, C, F, I, M, N>
2255where
2256    S: WorkerSourceFactory,
2257    F: TransformFactory<S::Sample> + Send + Sync + 'static,
2258    F::Transform: Send + 'static,
2259    TransformOutput<S, F>: Send + 'static,
2260    TransformFailure<S, F>: Send + 'static,
2261    F::Error: Send + 'static,
2262    C: Collate<TransformOutput<S, F>>,
2263    I: WorkerInit + Send + Sync + 'static,
2264    I::Error: Send + 'static,
2265    M: StreamMemoryPolicy + Send + 'static,
2266    N: StreamPinPolicy<S, C, F>,
2267{
2268    fn return_credit(&mut self, worker: usize) -> std::result::Result<(), ()> {
2269        self.pool.as_ref().ok_or(())?.pool().return_credit(worker)
2270    }
2271
2272    fn disarm_deadline(&self) {
2273        self.run_context.deadline.disarm();
2274    }
2275
2276    fn set_front_waiter(&mut self, worker: usize, sequence: Option<u64>) {
2277        if let Some(waiters) = M::waiters_mut(&mut self.front_waiters) {
2278            waiters[worker] = sequence;
2279        }
2280    }
2281
2282    fn close(&mut self, poisoned: bool) {
2283        self.exhausted = true;
2284        self.run_context.cancel();
2285        let mut credit_failed = false;
2286        while let Some((_, buffered)) = self.completed.pop() {
2287            credit_failed |= self.return_credit(buffered.worker).is_err();
2288        }
2289        if let Some(mut pool) = self.pool.take() {
2290            let persistent = pool.is_persistent();
2291            let quiesce_failed = persistent && pool.pool_mut().quiesce(self.generation).is_err();
2292            if poisoned || credit_failed || quiesce_failed {
2293                pool.pool_mut().poison();
2294            }
2295        }
2296        self.partial.clear();
2297    }
2298
2299    fn protocol_error(
2300        &mut self,
2301        sequence: Option<u64>,
2302        reason: impl Into<String>,
2303    ) -> StreamLoaderError<S, C, F, I> {
2304        self.disarm_deadline();
2305        self.close(false);
2306        LoaderError::StreamProtocol {
2307            sequence,
2308            reason: reason.into(),
2309        }
2310    }
2311
2312    fn consume(
2313        &mut self,
2314        buffered: BufferedRecord<TransformOutput<S, F>, M::Permit>,
2315    ) -> std::result::Result<bool, StreamLoaderError<S, C, F, I>> {
2316        let BufferedRecord {
2317            worker,
2318            record,
2319            permit,
2320        } = buffered;
2321        let next_sequence = if self.ordered {
2322            let Some(next) = record
2323                .sequence
2324                .expect("ordered records are validated")
2325                .checked_next()
2326            else {
2327                self.run_context.cancel();
2328                if self.return_credit(worker).is_err() {
2329                    self.close(true);
2330                    return Err(LoaderError::ChannelClosed {
2331                        batch: self.next_batch,
2332                    });
2333                }
2334                return Err(self.protocol_error(
2335                    Some(self.next_sequence),
2336                    "stream sequence identifier overflowed",
2337                ));
2338            };
2339            Some(next.get())
2340        } else {
2341            None
2342        };
2343        if self.return_credit(worker).is_err() {
2344            self.close(true);
2345            return Err(LoaderError::ChannelClosed {
2346                batch: self.next_batch,
2347            });
2348        }
2349        if let Some(next_sequence) = next_sequence {
2350            self.next_sequence = next_sequence;
2351        }
2352        drop(permit);
2353        self.partial.push(record.sample);
2354        Ok(self.partial.len() == self.batch_size)
2355    }
2356
2357    fn finish_batch(&mut self) -> std::result::Result<C::Batch, StreamLoaderError<S, C, F, I>> {
2358        let mut replacement = Vec::new();
2359        if replacement.try_reserve_exact(self.batch_size).is_err() {
2360            self.close(false);
2361            return Err(LoaderError::Configuration(invalid_configuration(
2362                "batch_size",
2363                "stream batch allocation is unavailable",
2364            )));
2365        }
2366        let samples = std::mem::replace(&mut self.partial, replacement);
2367        let batch = self.next_batch;
2368        let result = match catch_unwind(AssertUnwindSafe(|| self.collator.collate(samples))) {
2369            Ok(result) => result.map_err(|source| LoaderError::Pipeline {
2370                batch: Some(batch),
2371                worker: None,
2372                source: PipelineError::Collate(source),
2373            }),
2374            Err(_) => Err(LoaderError::CoordinatorPanic {
2375                stage: "stream collation",
2376                batch: Some(batch),
2377            }),
2378        }
2379        .and_then(|value| match self.pin_memory_status {
2380            PinMemoryStatus::Enabled(device) => {
2381                N::pin(value, device).map_err(|source| LoaderError::PinMemory { batch, source })
2382            }
2383            PinMemoryStatus::Disabled | PinMemoryStatus::DisabledNoAccelerator => Ok(value),
2384        });
2385        match result {
2386            Ok(batch) => match self.next_batch.checked_add(1) {
2387                Some(next) => {
2388                    self.next_batch = next;
2389                    self.disarm_deadline();
2390                    Ok(batch)
2391                }
2392                None => {
2393                    self.close(false);
2394                    Err(LoaderError::Configuration(invalid_configuration(
2395                        "batch_size",
2396                        "stream batch sequence overflowed",
2397                    )))
2398                }
2399            },
2400            Err(error) => {
2401                self.disarm_deadline();
2402                self.close(false);
2403                Err(error)
2404            }
2405        }
2406    }
2407
2408    fn map_failure(
2409        &mut self,
2410        worker: usize,
2411        sequence: Option<u64>,
2412        logical_id: Option<u64>,
2413        failure: StreamFailureFor<S, F, I, M>,
2414    ) -> StreamLoaderError<S, C, F, I> {
2415        let batch = Some(self.next_batch);
2416        match failure {
2417            StreamFailure::Source(source) => LoaderError::StreamPipeline {
2418                batch,
2419                worker,
2420                sequence,
2421                logical_id,
2422                source: PipelineError::Source(source),
2423            },
2424            StreamFailure::Transform(source) => LoaderError::StreamPipeline {
2425                batch,
2426                worker,
2427                sequence,
2428                logical_id,
2429                source: PipelineError::Transform(source),
2430            },
2431            StreamFailure::TransformInit(source) => LoaderError::StreamPipeline {
2432                batch: None,
2433                worker,
2434                sequence: None,
2435                logical_id: None,
2436                source: PipelineError::TransformInit(source),
2437            },
2438            StreamFailure::WorkerInit(source) => LoaderError::StreamPipeline {
2439                batch: None,
2440                worker,
2441                sequence: None,
2442                logical_id: None,
2443                source: PipelineError::WorkerInit(source),
2444            },
2445            StreamFailure::Memory(failure) => M::map_failure(failure, worker, sequence, logical_id),
2446            StreamFailure::Panic => LoaderError::StreamWorkerPanic {
2447                worker,
2448                batch,
2449                sequence,
2450                logical_id,
2451            },
2452        }
2453    }
2454}
2455
2456impl<S, C, F, I, M, N> Iterator for StreamLoaderIter<'_, S, C, F, I, M, N>
2457where
2458    S: WorkerSourceFactory,
2459    F: TransformFactory<S::Sample> + Send + Sync + 'static,
2460    F::Transform: Send + 'static,
2461    TransformOutput<S, F>: Send + 'static,
2462    TransformFailure<S, F>: Send + 'static,
2463    F::Error: Send + 'static,
2464    C: Collate<TransformOutput<S, F>>,
2465    I: WorkerInit + Send + Sync + 'static,
2466    I::Error: Send + 'static,
2467    M: StreamMemoryPolicy + Send + 'static,
2468    N: StreamPinPolicy<S, C, F>,
2469{
2470    type Item = std::result::Result<C::Batch, StreamLoaderError<S, C, F, I>>;
2471
2472    fn next(&mut self) -> Option<Self::Item> {
2473        if self.exhausted {
2474            return None;
2475        }
2476        if let Some(error) = self.pending_error.take() {
2477            self.close(false);
2478            return Some(Err(error));
2479        }
2480        if self.source_complete {
2481            self.close(false);
2482            return None;
2483        }
2484        let mut deadline_armed = false;
2485        loop {
2486            if self.ordered
2487                && let Some(index) = self
2488                    .completed
2489                    .iter()
2490                    .position(|(sequence, _)| *sequence == self.next_sequence)
2491            {
2492                let (_, buffered) = self.completed.swap_remove(index);
2493                match self.consume(buffered) {
2494                    Ok(true) => return Some(self.finish_batch()),
2495                    Ok(false) => continue,
2496                    Err(error) => return Some(Err(error)),
2497                }
2498            }
2499
2500            if self.ordered && self.completed.len() == self.reassembly_capacity {
2501                let expected = self.next_sequence;
2502                let found = self
2503                    .completed
2504                    .iter()
2505                    .map(|(sequence, _)| *sequence)
2506                    .min()
2507                    .expect("a full ordered reassembly window is nonempty");
2508                return Some(Err(self.protocol_error(
2509                    Some(expected),
2510                    format!(
2511                        "missing sequence {expected}; bounded reassembly window is full before buffered sequence {found}"
2512                    ),
2513                )));
2514            }
2515
2516            if self.ordered
2517                && self.run_context.byte_budget.is_some()
2518                && let Some(pool) = self.pool.as_ref()
2519                && let Some(front_waiters) = M::waiters(&self.front_waiters)
2520                && front_waiters
2521                    .iter()
2522                    .enumerate()
2523                    .any(|(worker, _)| !pool.pool().terminal.get(worker).copied().unwrap_or(false))
2524                && front_waiters.iter().enumerate().all(|(worker, sequence)| {
2525                    pool.pool().terminal.get(worker).copied().unwrap_or(false)
2526                        || sequence.is_some_and(|sequence| sequence > self.next_sequence)
2527                })
2528            {
2529                let expected = self.next_sequence;
2530                let found = M::waiters(&self.front_waiters)
2531                    .expect("enabled byte accounting has waiter state")
2532                    .iter()
2533                    .flatten()
2534                    .copied()
2535                    .min()
2536                    .expect("a nonterminal byte-budget worker has a front waiter");
2537                return Some(Err(self.protocol_error(
2538                    Some(expected),
2539                    format!(
2540                        "missing sequence {expected}; all active byte-budget workers are waiting at sequence {found} or later"
2541                    ),
2542                )));
2543            }
2544
2545            if self
2546                .pool
2547                .as_ref()
2548                .is_some_and(|pool| pool.pool().all_terminal())
2549            {
2550                if self.ordered
2551                    && let Some(found) = self.completed.iter().map(|(sequence, _)| *sequence).min()
2552                {
2553                    let expected = self.next_sequence;
2554                    return Some(Err(self.protocol_error(
2555                        Some(expected),
2556                        format!("missing sequence {expected} before buffered sequence {found}"),
2557                    )));
2558                }
2559                if self.ordered
2560                    && let Some(expected) = self.expected_records
2561                    && self.next_sequence != expected as u64
2562                {
2563                    let missing = self.next_sequence.min(expected as u64);
2564                    return Some(Err(self.protocol_error(
2565                        Some(missing),
2566                        format!(
2567                            "stream ended after {} ordered records, expected {expected}",
2568                            self.next_sequence
2569                        ),
2570                    )));
2571                }
2572                self.source_complete = true;
2573                if self.partial.is_empty() || self.drop_last {
2574                    self.close(false);
2575                    return None;
2576                }
2577                return Some(self.finish_batch());
2578            }
2579
2580            if !deadline_armed && let Some(timeout) = self.timeout {
2581                self.run_context.deadline.arm(timeout);
2582                deadline_armed = true;
2583            }
2584            let remaining = self
2585                .timeout
2586                .and_then(|_| self.run_context.deadline.remaining());
2587            let received = self
2588                .pool
2589                .as_ref()
2590                .expect("active stream iterator has a pool")
2591                .pool()
2592                .receive(&self.run_context, remaining);
2593            let completion = match received {
2594                StreamReceive::Completion(completion) => completion,
2595                StreamReceive::Timeout => {
2596                    let batch = self.next_batch;
2597                    self.run_context.cancel();
2598                    self.disarm_deadline();
2599                    self.close(false);
2600                    return Some(Err(LoaderError::Timeout { batch }));
2601                }
2602                StreamReceive::Cancelled => {
2603                    self.disarm_deadline();
2604                    self.close(false);
2605                    return Some(Err(LoaderError::Cancelled));
2606                }
2607                StreamReceive::Closed => {
2608                    let batch = self.next_batch;
2609                    self.disarm_deadline();
2610                    self.close(true);
2611                    return Some(Err(LoaderError::ChannelClosed { batch }));
2612                }
2613            };
2614            if completion.generation != self.generation {
2615                if completion.holds_credit && self.return_credit(completion.worker).is_err() {
2616                    self.close(true);
2617                    return Some(Err(LoaderError::ChannelClosed {
2618                        batch: self.next_batch,
2619                    }));
2620                }
2621                continue;
2622            }
2623
2624            match completion.result {
2625                Err(failure) => {
2626                    self.set_front_waiter(completion.worker, None);
2627                    self.run_context.cancel();
2628                    let fatal = matches!(
2629                        &failure,
2630                        StreamFailure::TransformInit(_)
2631                            | StreamFailure::WorkerInit(_)
2632                            | StreamFailure::Panic
2633                    );
2634                    if self
2635                        .pool
2636                        .as_mut()
2637                        .expect("stream pool")
2638                        .pool_mut()
2639                        .account_parts(completion.worker, completion.holds_credit, fatal, fatal)
2640                        .is_err()
2641                    {
2642                        self.close(true);
2643                        return Some(Err(LoaderError::ChannelClosed {
2644                            batch: self.next_batch,
2645                        }));
2646                    }
2647                    let error = self.map_failure(
2648                        completion.worker,
2649                        completion.sequence,
2650                        completion.logical_id,
2651                        failure,
2652                    );
2653                    self.disarm_deadline();
2654                    self.close(fatal);
2655                    return Some(Err(error));
2656                }
2657                Ok(StreamMessage::End) => {
2658                    self.set_front_waiter(completion.worker, None);
2659                    if self
2660                        .pool
2661                        .as_mut()
2662                        .expect("stream pool")
2663                        .pool_mut()
2664                        .account_parts(completion.worker, completion.holds_credit, true, false)
2665                        .is_err()
2666                    {
2667                        self.close(true);
2668                        return Some(Err(LoaderError::ChannelClosed {
2669                            batch: self.next_batch,
2670                        }));
2671                    }
2672                }
2673                Ok(StreamMessage::Waiting(waiting)) => {
2674                    self.set_front_waiter(completion.worker, Some(M::waiting_sequence(waiting)));
2675                }
2676                Ok(StreamMessage::Record(record, permit)) => {
2677                    self.set_front_waiter(completion.worker, None);
2678                    let buffered = BufferedRecord {
2679                        worker: completion.worker,
2680                        record,
2681                        permit,
2682                    };
2683                    if !self.ordered {
2684                        match self.consume(buffered) {
2685                            Ok(true) => return Some(self.finish_batch()),
2686                            Ok(false) => {}
2687                            Err(error) => return Some(Err(error)),
2688                        }
2689                        continue;
2690                    }
2691                    let Some(sequence) = buffered.record.sequence else {
2692                        self.run_context.cancel();
2693                        if self.return_credit(buffered.worker).is_err() {
2694                            self.close(true);
2695                            return Some(Err(LoaderError::ChannelClosed {
2696                                batch: self.next_batch,
2697                            }));
2698                        }
2699                        return Some(Err(self.protocol_error(
2700                            None,
2701                            "ordered stream record did not provide a sequence identifier",
2702                        )));
2703                    };
2704                    let sequence = sequence.get();
2705                    if sequence < self.next_sequence {
2706                        self.run_context.cancel();
2707                        if self.return_credit(buffered.worker).is_err() {
2708                            self.close(true);
2709                            return Some(Err(LoaderError::ChannelClosed {
2710                                batch: self.next_batch,
2711                            }));
2712                        }
2713                        return Some(Err(self.protocol_error(
2714                            Some(sequence),
2715                            format!(
2716                                "duplicate or past sequence {sequence}; next expected is {}",
2717                                self.next_sequence
2718                            ),
2719                        )));
2720                    }
2721                    if self
2722                        .completed
2723                        .iter()
2724                        .any(|(buffered_sequence, _)| *buffered_sequence == sequence)
2725                    {
2726                        self.run_context.cancel();
2727                        if self.return_credit(buffered.worker).is_err() {
2728                            self.close(true);
2729                            return Some(Err(LoaderError::ChannelClosed {
2730                                batch: self.next_batch,
2731                            }));
2732                        }
2733                        return Some(Err(self.protocol_error(
2734                            Some(sequence),
2735                            format!("duplicate buffered sequence {sequence}"),
2736                        )));
2737                    }
2738                    self.completed.push((sequence, buffered));
2739                }
2740            }
2741        }
2742    }
2743}
2744
2745impl<S, C, F, I, M, N> Drop for StreamLoaderIter<'_, S, C, F, I, M, N>
2746where
2747    S: WorkerSourceFactory,
2748    F: TransformFactory<S::Sample>,
2749    C: Collate<TransformOutput<S, F>>,
2750    I: WorkerInit,
2751    M: StreamMemoryPolicy,
2752{
2753    fn drop(&mut self) {
2754        self.run_context.cancel();
2755        let mut failed = false;
2756        while let Some((_, buffered)) = self.completed.pop() {
2757            failed |= self
2758                .pool
2759                .as_ref()
2760                .is_none_or(|pool| pool.pool().return_credit(buffered.worker).is_err());
2761        }
2762        if let Some(mut pool) = self.pool.take()
2763            && pool.is_persistent()
2764            && (failed || pool.pool_mut().quiesce(self.generation).is_err())
2765        {
2766            pool.pool_mut().poison();
2767        }
2768    }
2769}
2770
2771fn invalid_configuration(field: &'static str, reason: impl Into<String>) -> RustTorchError {
2772    RustTorchError::InvalidConfiguration {
2773        field,
2774        reason: reason.into(),
2775    }
2776}
2777
2778#[cfg(test)]
2779mod tests {
2780    use std::convert::Infallible;
2781
2782    use super::*;
2783
2784    type TestCompletion = StreamCompletion<
2785        usize,
2786        (),
2787        Infallible,
2788        Infallible,
2789        Infallible,
2790        Infallible,
2791        Infallible,
2792        Infallible,
2793    >;
2794    type TestResult = StreamResult<
2795        usize,
2796        (),
2797        Infallible,
2798        Infallible,
2799        Infallible,
2800        Infallible,
2801        Infallible,
2802        Infallible,
2803    >;
2804
2805    fn ready_completion(result: TestResult) -> TestCompletion {
2806        StreamCompletion {
2807            worker: 0,
2808            generation: 0,
2809            sequence: None,
2810            logical_id: None,
2811            holds_credit: false,
2812            result,
2813        }
2814    }
2815
2816    fn receive_after_expiry(completion: TestCompletion) -> TestCompletion {
2817        let (sender, receiver) = bounded(1);
2818        sender.send(completion).unwrap();
2819        let context = WorkerRunContext::new(0, 0, 0);
2820        context.deadline.arm(Duration::ZERO);
2821        let shutdown = CancellationToken::new();
2822        match receive_ready_first(&receiver, &context, &shutdown, Some(Duration::ZERO)) {
2823            ReadyFirst::Completion(completion) => completion,
2824            ReadyFirst::Timeout => panic!("ready completion lost to expired deadline"),
2825            ReadyFirst::Cancelled => panic!("ready completion lost to cancellation"),
2826            ReadyFirst::Closed => panic!("ready completion lost to channel close"),
2827        }
2828    }
2829
2830    #[test]
2831    fn queued_error_and_end_beat_an_expired_receive_deadline() {
2832        let error = receive_after_expiry(ready_completion(Err(StreamFailure::Panic)));
2833        assert!(matches!(error.result, Err(StreamFailure::Panic)));
2834
2835        let end = receive_after_expiry(ready_completion(Ok(StreamMessage::End)));
2836        assert!(matches!(end.result, Ok(StreamMessage::End)));
2837    }
2838}