1use std::{
76 convert::Infallible,
77 marker::PhantomData,
78 mem::{MaybeUninit, size_of},
79 num::NonZeroUsize,
80 panic::{AssertUnwindSafe, catch_unwind},
81 sync::{Arc, Mutex, atomic::AtomicUsize},
82 thread::{self, JoinHandle},
83 time::Duration,
84};
85
86use crossbeam_channel::{Receiver, RecvError, Sender, TryRecvError, after, bounded, select};
87
88use rusttorch_core::{Device, Result, RustTorchError, available_devices};
89
90use crate::memory::{BudgetError, ByteBudget, MemoryDisabled, MemoryEnabled, MemoryPolicy};
91use crate::worker::WorkerRunContext;
92#[path = "stream_checkpoint.rs"]
93mod exact;
94use crate::{
95 Auto, CancellationToken, CloneTransformFactory, Collate, Deadline, DefaultCollator, Explicit,
96 IdentityTransformFactory, LoaderError, MemoryFootprint, NoWorkerInit, PinDisabled, PinEnabled,
97 PinMemory, PinMemoryStatus, PipelineError, TaskContext, Transform, TransformFactory,
98 WorkerContext, WorkerInfo, WorkerInit, with_worker_info,
99};
100pub use exact::{ExactStreamDataLoader, ExactStreamLoaderIter};
101
102#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
104pub struct SequenceId(u64);
105
106impl SequenceId {
107 pub const fn new(value: u64) -> Self {
109 Self(value)
110 }
111
112 pub const fn get(self) -> u64 {
114 self.0
115 }
116
117 pub const fn checked_next(self) -> Option<Self> {
119 match self.0.checked_add(1) {
120 Some(value) => Some(Self(value)),
121 None => None,
122 }
123 }
124}
125
126#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
128pub struct LogicalSampleId(u64);
129
130impl LogicalSampleId {
131 pub const fn new(value: u64) -> Self {
133 Self(value)
134 }
135
136 pub const fn get(self) -> u64 {
138 self.0
139 }
140
141 pub const fn checked_next(self) -> Option<Self> {
143 match self.0.checked_add(1) {
144 Some(value) => Some(Self(value)),
145 None => None,
146 }
147 }
148}
149
150#[derive(Clone, Debug, Eq, PartialEq)]
152pub struct WorkerRecord<T> {
153 pub sequence: Option<SequenceId>,
155 pub logical_id: LogicalSampleId,
157 pub sample: T,
159}
160
161pub trait WorkerSourceFactory: Send + Sync + 'static {
163 type Sample: Send + 'static;
165 type Error: Send + 'static;
167 type Source: Iterator<Item = std::result::Result<WorkerRecord<Self::Sample>, Self::Error>>
169 + Send
170 + 'static;
171
172 fn create(&self, worker: WorkerContext) -> std::result::Result<Self::Source, Self::Error>;
174
175 fn exact_len(&self) -> Option<usize> {
177 None
178 }
179}
180
181type TransformOutput<S, F> =
182 <<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<
183 <S as WorkerSourceFactory>::Sample,
184 >>::Output;
185type TransformFailure<S, F> =
186 <<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<
187 <S as WorkerSourceFactory>::Sample,
188 >>::Error;
189type StreamPipelineError<S, C, F, I> = PipelineError<
190 <S as WorkerSourceFactory>::Error,
191 TransformFailure<S, F>,
192 <C as Collate<TransformOutput<S, F>>>::Error,
193 <F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Error,
194 <I as WorkerInit>::Error,
195>;
196type StreamLoaderError<S, C, F, I> = LoaderError<StreamPipelineError<S, C, F, I>>;
197type ReassemblyEntry<T, P> = (u64, BufferedRecord<T, P>);
198type RetainedReassembly<T, P> = Mutex<Vec<ReassemblyEntry<T, P>>>;
199type StreamFootprint<S, F> = fn(&TransformOutput<S, F>) -> usize;
200
201enum StreamByteFailure {
202 MemoryLimit {
203 limit: usize,
204 actual: usize,
205 },
206 Protocol {
207 sequence: Option<u64>,
208 reason: String,
209 },
210}
211
212trait StreamMemoryPolicy: MemoryPolicy {
213 type Waiters;
214 type Waiting: Send + 'static;
215 type Failure: Send + 'static;
216
217 fn waiter_bytes(workers: usize) -> Result<usize>;
218 fn preflight_waiters(workers: usize) -> Result<()>;
219 fn new_waiters(workers: usize) -> Result<Self::Waiters>;
220 fn waiters(waiters: &Self::Waiters) -> Option<&[Option<u64>]>;
221 fn waiters_mut(waiters: &mut Self::Waiters) -> Option<&mut [Option<u64>]>;
222 fn waiting(sequence: u64) -> Option<Self::Waiting>;
223 fn waiting_sequence(waiting: Self::Waiting) -> u64;
224 fn memory_limit(limit: usize, actual: usize) -> Option<Self::Failure>;
225 fn protocol(sequence: Option<u64>, reason: String) -> Option<Self::Failure>;
226 fn map_failure<E>(
227 failure: Self::Failure,
228 worker: usize,
229 sequence: Option<u64>,
230 logical_id: Option<u64>,
231 ) -> LoaderError<E>;
232}
233
234impl StreamMemoryPolicy for MemoryDisabled {
235 type Waiters = ();
236 type Waiting = Infallible;
237 type Failure = Infallible;
238
239 fn waiter_bytes(_workers: usize) -> Result<usize> {
240 Ok(0)
241 }
242
243 fn preflight_waiters(_workers: usize) -> Result<()> {
244 Ok(())
245 }
246
247 fn new_waiters(_workers: usize) -> Result<Self::Waiters> {
248 Ok(())
249 }
250
251 fn waiters(_waiters: &Self::Waiters) -> Option<&[Option<u64>]> {
252 None
253 }
254
255 fn waiters_mut(_waiters: &mut Self::Waiters) -> Option<&mut [Option<u64>]> {
256 None
257 }
258
259 fn waiting(_sequence: u64) -> Option<Self::Waiting> {
260 None
261 }
262
263 fn waiting_sequence(waiting: Self::Waiting) -> u64 {
264 match waiting {}
265 }
266
267 fn memory_limit(_limit: usize, _actual: usize) -> Option<Self::Failure> {
268 None
269 }
270
271 fn protocol(_sequence: Option<u64>, _reason: String) -> Option<Self::Failure> {
272 None
273 }
274
275 fn map_failure<E>(
276 failure: Self::Failure,
277 _worker: usize,
278 _sequence: Option<u64>,
279 _logical_id: Option<u64>,
280 ) -> LoaderError<E> {
281 match failure {}
282 }
283}
284
285impl StreamMemoryPolicy for MemoryEnabled {
286 type Waiters = Vec<Option<u64>>;
287 type Waiting = u64;
288 type Failure = StreamByteFailure;
289
290 fn waiter_bytes(workers: usize) -> Result<usize> {
291 workers
292 .checked_mul(size_of::<Option<u64>>())
293 .ok_or_else(|| capacity_error("stream byte waiter storage exceeds usize"))
294 }
295
296 fn preflight_waiters(workers: usize) -> Result<()> {
297 preflight_vec::<Option<u64>>(workers, "stream byte waiter state")
298 }
299
300 fn new_waiters(workers: usize) -> Result<Self::Waiters> {
301 let mut waiters = Vec::new();
302 reserve_exact(&mut waiters, workers, "stream byte waiter state")?;
303 waiters.resize(workers, None);
304 Ok(waiters)
305 }
306
307 fn waiters(waiters: &Self::Waiters) -> Option<&[Option<u64>]> {
308 Some(waiters)
309 }
310
311 fn waiters_mut(waiters: &mut Self::Waiters) -> Option<&mut [Option<u64>]> {
312 Some(waiters)
313 }
314
315 fn waiting(sequence: u64) -> Option<Self::Waiting> {
316 Some(sequence)
317 }
318
319 fn waiting_sequence(waiting: Self::Waiting) -> u64 {
320 waiting
321 }
322
323 fn memory_limit(limit: usize, actual: usize) -> Option<Self::Failure> {
324 Some(StreamByteFailure::MemoryLimit { limit, actual })
325 }
326
327 fn protocol(sequence: Option<u64>, reason: String) -> Option<Self::Failure> {
328 Some(StreamByteFailure::Protocol { sequence, reason })
329 }
330
331 fn map_failure<E>(
332 failure: Self::Failure,
333 worker: usize,
334 sequence: Option<u64>,
335 logical_id: Option<u64>,
336 ) -> LoaderError<E> {
337 match failure {
338 StreamByteFailure::MemoryLimit { limit, actual } => LoaderError::MemoryLimit {
339 batch: None,
340 worker: Some(worker),
341 sequence,
342 logical_id,
343 limit,
344 actual,
345 },
346 StreamByteFailure::Protocol { sequence, reason } => {
347 LoaderError::StreamProtocol { sequence, reason }
348 }
349 }
350 }
351}
352
353#[doc(hidden)]
354pub trait StreamPinPolicy<S, C, F>
355where
356 S: WorkerSourceFactory,
357 F: TransformFactory<S::Sample>,
358 C: Collate<TransformOutput<S, F>>,
359{
360 fn pin(batch: C::Batch, device: Device) -> Result<C::Batch>;
361}
362
363impl<S, C, F> StreamPinPolicy<S, C, F> for PinDisabled
364where
365 S: WorkerSourceFactory,
366 F: TransformFactory<S::Sample>,
367 C: Collate<TransformOutput<S, F>>,
368{
369 fn pin(batch: C::Batch, _device: Device) -> Result<C::Batch> {
370 Ok(batch)
371 }
372}
373
374impl<S, C, F, Q> StreamPinPolicy<S, C, F> for PinEnabled<Q>
375where
376 S: WorkerSourceFactory,
377 F: TransformFactory<S::Sample>,
378 C: Collate<TransformOutput<S, F>>,
379 C::Batch: PinMemory,
380{
381 fn pin(batch: C::Batch, device: Device) -> Result<C::Batch> {
382 batch.pin_memory(device)
383 }
384}
385
386type StreamCompletionFor<S, F, I, M> = StreamCompletion<
387 TransformOutput<S, F>,
388 <M as MemoryPolicy>::Permit,
389 <M as StreamMemoryPolicy>::Waiting,
390 <M as StreamMemoryPolicy>::Failure,
391 <S as WorkerSourceFactory>::Error,
392 TransformFailure<S, F>,
393 <F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Error,
394 <I as WorkerInit>::Error,
395>;
396type StreamFailureFor<S, F, I, M> = StreamFailure<
397 <S as WorkerSourceFactory>::Error,
398 TransformFailure<S, F>,
399 <F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Error,
400 <I as WorkerInit>::Error,
401 <M as StreamMemoryPolicy>::Failure,
402>;
403type StreamResult<T, P, W, ME, SE, TE, FE, IE> =
404 std::result::Result<StreamMessage<T, P, W>, StreamFailure<SE, TE, FE, IE, ME>>;
405type StreamCompletionSender<T, P, W, ME, SE, TE, FE, IE> =
406 Sender<StreamCompletion<T, P, W, ME, SE, TE, FE, IE>>;
407
408enum StreamFailure<SE, TE, FE, IE, ME> {
409 Source(SE),
410 Transform(TE),
411 TransformInit(FE),
412 WorkerInit(IE),
413 Memory(ME),
414 Panic,
415}
416
417enum StreamMessage<T, P, W> {
418 Record(WorkerRecord<T>, P),
419 Waiting(W),
420 End,
421}
422
423struct StreamCompletion<T, P, W, ME, SE, TE, FE, IE> {
424 worker: usize,
425 generation: u64,
426 sequence: Option<u64>,
427 logical_id: Option<u64>,
428 holds_credit: bool,
429 result: StreamResult<T, P, W, ME, SE, TE, FE, IE>,
430}
431
432enum StreamControl {
433 Begin(WorkerRunContext),
434}
435
436enum StreamReceive<S, F, I, M>
437where
438 S: WorkerSourceFactory,
439 F: TransformFactory<S::Sample>,
440 I: WorkerInit,
441 M: StreamMemoryPolicy,
442{
443 Completion(StreamCompletionFor<S, F, I, M>),
444 Timeout,
445 Cancelled,
446 Closed,
447}
448
449enum ReadyFirst<T> {
450 Completion(T),
451 Timeout,
452 Cancelled,
453 Closed,
454}
455
456const MAX_STREAM_QUEUE_ALLOCATION_BYTES: usize = 64 * 1024 * 1024;
457const CHANNEL_CONTROL_BLOCK_ALLOWANCE_BYTES: usize = 2 * 1024;
458
459#[allow(dead_code)]
460#[repr(C)]
461struct CapacitySlot<T> {
462 stamp: AtomicUsize,
463 message: MaybeUninit<T>,
464}
465
466fn validate_stream_capacity<S, F, I, M>(
467 workers: usize,
468 prefetch_factor: usize,
469 outstanding: usize,
470 batch_size: usize,
471 reassembly_slots: usize,
472) -> Result<()>
473where
474 S: WorkerSourceFactory,
475 F: TransformFactory<S::Sample>,
476 I: WorkerInit,
477 M: StreamMemoryPolicy,
478{
479 validate_stream_capacity_extra::<S, F, I, M>(
480 workers,
481 prefetch_factor,
482 outstanding,
483 batch_size,
484 reassembly_slots,
485 0,
486 )
487}
488
489fn validate_stream_capacity_extra<S, F, I, M>(
490 workers: usize,
491 prefetch_factor: usize,
492 outstanding: usize,
493 batch_size: usize,
494 reassembly_slots: usize,
495 extra: usize,
496) -> Result<()>
497where
498 S: WorkerSourceFactory,
499 F: TransformFactory<S::Sample>,
500 I: WorkerInit,
501 M: StreamMemoryPolicy,
502{
503 let expected = workers
504 .checked_mul(prefetch_factor)
505 .ok_or_else(|| capacity_error("workers multiplied by prefetch_factor exceeds usize"))?;
506 if expected != outstanding {
507 return Err(capacity_error(
508 "outstanding capacity does not match workers multiplied by prefetch_factor",
509 ));
510 }
511 for capacity in [1, prefetch_factor, outstanding] {
512 capacity
513 .checked_add(1)
514 .and_then(usize::checked_next_power_of_two)
515 .and_then(|mark_bit| mark_bit.checked_mul(2))
516 .ok_or_else(|| capacity_error("bounded channel ring arithmetic overflowed"))?;
517 }
518
519 let credit_bytes = outstanding
520 .checked_mul(size_of::<CapacitySlot<()>>())
521 .ok_or_else(|| capacity_error("stream credit storage exceeds usize"))?;
522 let result_bytes = outstanding
523 .checked_mul(size_of::<CapacitySlot<StreamCompletionFor<S, F, I, M>>>())
524 .ok_or_else(|| capacity_error("stream result storage exceeds usize"))?;
525 let control_bytes = workers
526 .checked_mul(size_of::<CapacitySlot<StreamControl>>())
527 .ok_or_else(|| capacity_error("stream control storage exceeds usize"))?;
528 let batch_bytes = batch_size
529 .checked_mul(size_of::<TransformOutput<S, F>>())
530 .and_then(|bytes| bytes.checked_mul(2))
531 .ok_or_else(|| capacity_error("stream batch storage exceeds usize"))?;
532 let reassembly_bytes = reassembly_slots
533 .checked_mul(size_of::<
534 ReassemblyEntry<TransformOutput<S, F>, <M as MemoryPolicy>::Permit>,
535 >())
536 .ok_or_else(|| capacity_error("ordered stream reassembly exceeds usize"))?;
537 let per_worker = size_of::<WorkerInfo>()
538 .checked_add(size_of::<(Sender<()>, Receiver<()>)>())
539 .and_then(|bytes| {
540 bytes.checked_add(size_of::<(Sender<StreamControl>, Receiver<StreamControl>)>())
541 })
542 .and_then(|bytes| bytes.checked_add(size_of::<Sender<()>>()))
543 .and_then(|bytes| bytes.checked_add(size_of::<Sender<StreamControl>>()))
544 .and_then(|bytes| bytes.checked_add(size_of::<JoinHandle<()>>()))
545 .and_then(|bytes| bytes.checked_add(size_of::<bool>()))
546 .ok_or_else(|| capacity_error("stream worker bookkeeping exceeds usize"))?;
547 let waiter_bytes = M::waiter_bytes(workers)?;
548 let bookkeeping = workers
549 .checked_mul(per_worker)
550 .and_then(|bytes| bytes.checked_add(waiter_bytes))
551 .ok_or_else(|| capacity_error("stream worker bookkeeping exceeds usize"))?;
552 let channel_count = workers
553 .checked_mul(2)
554 .and_then(|count| count.checked_add(1))
555 .ok_or_else(|| capacity_error("stream channel count exceeds usize"))?;
556 let channel_bytes = channel_count
557 .checked_mul(CHANNEL_CONTROL_BLOCK_ALLOWANCE_BYTES)
558 .ok_or_else(|| capacity_error("stream channel controls exceed usize"))?;
559 let aggregate = credit_bytes
560 .checked_add(result_bytes)
561 .and_then(|bytes| bytes.checked_add(control_bytes))
562 .and_then(|bytes| bytes.checked_add(batch_bytes))
563 .and_then(|bytes| bytes.checked_add(reassembly_bytes))
564 .and_then(|bytes| bytes.checked_add(bookkeeping))
565 .and_then(|bytes| bytes.checked_add(channel_bytes))
566 .and_then(|bytes| bytes.checked_add(extra))
567 .ok_or_else(|| capacity_error("aggregate stream queue storage exceeds usize"))?;
568 if aggregate > MAX_STREAM_QUEUE_ALLOCATION_BYTES {
569 return Err(capacity_error(format!(
570 "aggregate stream queue storage requires {aggregate} bytes, above the {MAX_STREAM_QUEUE_ALLOCATION_BYTES}-byte safety ceiling"
571 )));
572 }
573 preflight_channel_storage::<()>(outstanding)?;
574 preflight_channel_storage::<StreamControl>(workers)?;
575 preflight_channel_storage::<StreamCompletionFor<S, F, I, M>>(outstanding)?;
576 preflight_vec::<TransformOutput<S, F>>(batch_size, "stream batch")?;
577 M::preflight_waiters(workers)?;
578 Ok(())
579}
580
581fn preflight_channel_storage<T>(capacity: usize) -> Result<()> {
582 let mut reservation: Vec<MaybeUninit<CapacitySlot<T>>> = Vec::new();
583 reservation
584 .try_reserve_exact(capacity)
585 .map_err(|error| capacity_error(format!("bounded stream storage is unavailable: {error}")))
586}
587
588fn preflight_vec<T>(capacity: usize, label: &'static str) -> Result<()> {
589 let mut reservation: Vec<T> = Vec::new();
590 reservation
591 .try_reserve_exact(capacity)
592 .map_err(|error| capacity_error(format!("{label} capacity is unavailable: {error}")))
593}
594
595fn capacity_error(reason: impl Into<String>) -> RustTorchError {
596 invalid_configuration("prefetch_factor", reason)
597}
598
599struct StreamWorkerPool<S, F, I, M>
600where
601 S: WorkerSourceFactory,
602 F: TransformFactory<S::Sample>,
603 I: WorkerInit,
604 M: StreamMemoryPolicy,
605{
606 controls: Vec<Sender<StreamControl>>,
607 credits: Vec<Sender<()>>,
608 results: Option<Receiver<StreamCompletionFor<S, F, I, M>>>,
609 handles: Vec<JoinHandle<()>>,
610 shutdown: CancellationToken,
611 active: Option<WorkerRunContext>,
612 terminal: Vec<bool>,
613 terminal_count: usize,
614 fatal_exit: bool,
615 poisoned: bool,
616}
617
618impl<S, F, I, M> StreamWorkerPool<S, F, I, M>
619where
620 S: WorkerSourceFactory,
621 F: TransformFactory<S::Sample> + Send + Sync + 'static,
622 F::Transform: Send + 'static,
623 TransformOutput<S, F>: Send + 'static,
624 TransformFailure<S, F>: Send + 'static,
625 F::Error: Send + 'static,
626 I: WorkerInit + Send + Sync + 'static,
627 I::Error: Send + 'static,
628 M: StreamMemoryPolicy + Send + 'static,
629{
630 #[allow(clippy::too_many_arguments)]
631 fn new(
632 source_factory: Arc<S>,
633 transform_factory: Arc<F>,
634 initializer: Arc<I>,
635 configuration: StreamConfiguration,
636 generation: u64,
637 outstanding: usize,
638 reassembly_slots: usize,
639 footprint: Option<StreamFootprint<S, F>>,
640 ) -> Result<Self> {
641 validate_stream_capacity::<S, F, I, M>(
642 configuration.workers,
643 configuration.prefetch_factor,
644 outstanding,
645 configuration.batch_size,
646 reassembly_slots,
647 )?;
648
649 let (result_sender, result_receiver) =
650 bounded_checked(outstanding, "stream result channel")?;
651 let mut credit_lanes = Vec::new();
652 let mut control_lanes = Vec::new();
653 reserve_exact(
654 &mut credit_lanes,
655 configuration.workers,
656 "stream credit channels",
657 )?;
658 reserve_exact(
659 &mut control_lanes,
660 configuration.workers,
661 "stream control channels",
662 )?;
663 for _ in 0..configuration.workers {
664 let (credit_sender, credit_receiver) =
665 bounded_checked(configuration.prefetch_factor, "stream credit channel")?;
666 for _ in 0..configuration.prefetch_factor {
667 credit_sender.try_send(()).map_err(|_| {
668 capacity_error("validated stream credit channel rejected a token")
669 })?;
670 }
671 credit_lanes.push((credit_sender, credit_receiver));
672 control_lanes.push(bounded_checked(1, "stream control channel")?);
673 }
674
675 let shutdown = CancellationToken::new();
676 let mut pool = Self {
677 controls: Vec::new(),
678 credits: Vec::new(),
679 results: Some(result_receiver),
680 handles: Vec::new(),
681 shutdown: shutdown.clone(),
682 active: None,
683 terminal: Vec::new(),
684 terminal_count: 0,
685 fatal_exit: false,
686 poisoned: false,
687 };
688 reserve_exact(&mut pool.controls, configuration.workers, "stream controls")?;
689 reserve_exact(&mut pool.credits, configuration.workers, "stream credits")?;
690 reserve_exact(&mut pool.handles, configuration.workers, "stream workers")?;
691 reserve_exact(
692 &mut pool.terminal,
693 configuration.workers,
694 "stream terminal state",
695 )?;
696 pool.terminal.resize(configuration.workers, false);
697
698 for (id, ((credit_sender, credit_receiver), (control_sender, control_receiver))) in
699 credit_lanes.into_iter().zip(control_lanes).enumerate()
700 {
701 let info = WorkerInfo::from_loader_seed(
702 id,
703 configuration.workers,
704 configuration.loader_seed,
705 configuration.rank,
706 generation,
707 )?;
708 pool.credits.push(credit_sender);
709 pool.controls.push(control_sender);
710 let source_factory = Arc::clone(&source_factory);
711 let transform_factory = Arc::clone(&transform_factory);
712 let initializer = Arc::clone(&initializer);
713 let results = result_sender.clone();
714 let shutdown = shutdown.clone();
715 let handle = thread::Builder::new()
716 .name(format!("rusttorch-stream-worker-{id}"))
717 .spawn(move || {
718 run_stream_worker::<S, F, I, M>(
719 info,
720 generation,
721 source_factory,
722 transform_factory,
723 initializer,
724 control_receiver,
725 credit_receiver,
726 results,
727 shutdown,
728 footprint,
729 configuration.ordered,
730 );
731 })
732 .map_err(|error| RustTorchError::BackendUnavailable {
733 backend: "data loader stream worker threads",
734 reason: error.to_string(),
735 })?;
736 pool.handles.push(handle);
737 }
738 drop(result_sender);
739 Ok(pool)
740 }
741}
742
743impl<S, F, I, M> StreamWorkerPool<S, F, I, M>
744where
745 S: WorkerSourceFactory,
746 F: TransformFactory<S::Sample>,
747 I: WorkerInit,
748 M: StreamMemoryPolicy,
749{
750 fn start_generation(&mut self, context: WorkerRunContext) -> std::result::Result<(), ()> {
751 if self.poisoned || self.active.is_some() || self.shutdown.is_cancelled() {
752 return Err(());
753 }
754 self.terminal.fill(false);
755 self.terminal_count = 0;
756 self.fatal_exit = false;
757 self.active = Some(context.clone());
758 for control in &self.controls {
759 if control.send(StreamControl::Begin(context.clone())).is_err() {
760 self.poisoned = true;
761 self.shutdown();
762 return Err(());
763 }
764 }
765 Ok(())
766 }
767
768 fn receive(
769 &self,
770 context: &WorkerRunContext,
771 timeout: Option<Duration>,
772 ) -> StreamReceive<S, F, I, M> {
773 let results = self
774 .results
775 .as_ref()
776 .expect("stream result receiver exists before shutdown");
777 match receive_ready_first(results, context, &self.shutdown, timeout) {
778 ReadyFirst::Completion(completion) => StreamReceive::Completion(completion),
779 ReadyFirst::Timeout => StreamReceive::Timeout,
780 ReadyFirst::Cancelled => StreamReceive::Cancelled,
781 ReadyFirst::Closed => StreamReceive::Closed,
782 }
783 }
784
785 fn account(
786 &mut self,
787 completion: &StreamCompletionFor<S, F, I, M>,
788 ) -> std::result::Result<(), ()> {
789 let fatal = matches!(
790 &completion.result,
791 Err(StreamFailure::TransformInit(_)
792 | StreamFailure::WorkerInit(_)
793 | StreamFailure::Panic)
794 );
795 let terminal = fatal || matches!(&completion.result, Ok(StreamMessage::End));
796 self.account_parts(completion.worker, completion.holds_credit, terminal, fatal)
797 }
798
799 fn account_parts(
800 &mut self,
801 worker: usize,
802 holds_credit: bool,
803 terminal: bool,
804 fatal: bool,
805 ) -> std::result::Result<(), ()> {
806 if holds_credit {
807 self.return_credit(worker)?;
808 }
809 if fatal {
810 self.fatal_exit = true;
811 }
812 if terminal {
813 let Some(seen) = self.terminal.get_mut(worker) else {
814 return Err(());
815 };
816 if !*seen {
817 *seen = true;
818 self.terminal_count += 1;
819 }
820 }
821 Ok(())
822 }
823
824 fn return_credit(&self, worker: usize) -> std::result::Result<(), ()> {
825 let Some(credit) = self.credits.get(worker) else {
826 return Err(());
827 };
828 credit.try_send(()).map_err(|_| ())
829 }
830
831 fn all_terminal(&self) -> bool {
832 self.terminal_count == self.terminal.len()
833 }
834
835 fn quiesce(&mut self, generation: u64) -> std::result::Result<(), ()> {
836 let Some(active) = self.active.take() else {
837 return Ok(());
838 };
839 active.cancel();
840 while !self.all_terminal() {
841 let completion = self.results.as_ref().ok_or(())?.recv().map_err(|_| ())?;
842 if completion.generation == generation {
843 self.account(&completion)?;
844 } else if completion.holds_credit {
845 self.return_credit(completion.worker)?;
846 }
847 }
848 while let Some(results) = self.results.as_ref() {
849 match results.try_recv() {
850 Ok(completion) => self.account(&completion)?,
851 Err(TryRecvError::Empty) => break,
852 Err(TryRecvError::Disconnected) => return Err(()),
853 }
854 }
855 if self.fatal_exit {
856 self.poisoned = true;
857 self.shutdown();
858 return Err(());
859 }
860 Ok(())
861 }
862
863 fn poison(&mut self) {
864 self.poisoned = true;
865 self.shutdown();
866 }
867
868 fn shutdown(&mut self) {
869 if let Some(active) = self.active.take() {
870 active.cancel();
871 }
872 self.shutdown.cancel();
873 self.controls.clear();
874 self.credits.clear();
875 self.results.take();
876 for handle in self.handles.drain(..) {
877 let _ = handle.join();
878 }
879 }
880}
881
882impl<S, F, I, M> Drop for StreamWorkerPool<S, F, I, M>
883where
884 S: WorkerSourceFactory,
885 F: TransformFactory<S::Sample>,
886 I: WorkerInit,
887 M: StreamMemoryPolicy,
888{
889 fn drop(&mut self) {
890 self.shutdown();
891 }
892}
893
894fn receive_ready_first<T>(
895 results: &Receiver<T>,
896 context: &WorkerRunContext,
897 shutdown: &CancellationToken,
898 timeout: Option<Duration>,
899) -> ReadyFirst<T> {
900 match results.try_recv() {
901 Ok(completion) => return ReadyFirst::Completion(completion),
902 Err(TryRecvError::Disconnected) => return ReadyFirst::Closed,
903 Err(TryRecvError::Empty) => {}
904 }
905 if let Some(timeout) = timeout {
906 let timer = after(timeout);
907 select! {
908 recv(results) -> result => map_ready_receive(result),
909 recv(context.cancellation.signal()) -> _ => match results.try_recv() {
910 Ok(completion) => ReadyFirst::Completion(completion),
911 Err(TryRecvError::Empty) => ReadyFirst::Cancelled,
912 Err(TryRecvError::Disconnected) => ReadyFirst::Closed,
913 },
914 recv(shutdown.signal()) -> _ => ReadyFirst::Closed,
915 recv(timer) -> _ => match results.try_recv() {
916 Ok(completion) => ReadyFirst::Completion(completion),
917 Err(TryRecvError::Empty) => ReadyFirst::Timeout,
918 Err(TryRecvError::Disconnected) => ReadyFirst::Closed,
919 },
920 }
921 } else {
922 select! {
923 recv(results) -> result => map_ready_receive(result),
924 recv(context.cancellation.signal()) -> _ => match results.try_recv() {
925 Ok(completion) => ReadyFirst::Completion(completion),
926 Err(TryRecvError::Empty) => ReadyFirst::Cancelled,
927 Err(TryRecvError::Disconnected) => ReadyFirst::Closed,
928 },
929 recv(shutdown.signal()) -> _ => ReadyFirst::Closed,
930 }
931 }
932}
933
934fn map_ready_receive<T>(result: std::result::Result<T, RecvError>) -> ReadyFirst<T> {
935 match result {
936 Ok(completion) => ReadyFirst::Completion(completion),
937 Err(_) => ReadyFirst::Closed,
938 }
939}
940
941fn reserve_exact<T>(values: &mut Vec<T>, capacity: usize, label: &'static str) -> Result<()> {
942 values
943 .try_reserve_exact(capacity)
944 .map_err(|error| capacity_error(format!("{label} capacity is unavailable: {error}")))
945}
946
947fn bounded_checked<T>(capacity: usize, label: &'static str) -> Result<(Sender<T>, Receiver<T>)> {
948 catch_unwind(AssertUnwindSafe(|| bounded(capacity))).map_err(|_| {
949 capacity_error(format!(
950 "{label} rejected the validated capacity {capacity}"
951 ))
952 })
953}
954
955#[allow(clippy::too_many_arguments)]
956fn run_stream_worker<S, F, I, M>(
957 info: WorkerInfo,
958 initialization_generation: u64,
959 source_factory: Arc<S>,
960 transform_factory: Arc<F>,
961 initializer: Arc<I>,
962 controls: Receiver<StreamControl>,
963 credits: Receiver<()>,
964 results: Sender<StreamCompletionFor<S, F, I, M>>,
965 shutdown: CancellationToken,
966 footprint: Option<StreamFootprint<S, F>>,
967 ordered: bool,
968) where
969 S: WorkerSourceFactory,
970 F: TransformFactory<S::Sample> + Send + Sync + 'static,
971 F::Transform: Send + 'static,
972 TransformOutput<S, F>: Send + 'static,
973 TransformFailure<S, F>: Send + 'static,
974 F::Error: Send + 'static,
975 I: WorkerInit + Send + Sync + 'static,
976 I::Error: Send + 'static,
977 M: StreamMemoryPolicy + Send + 'static,
978{
979 let lifecycle = WorkerContext::new(info, shutdown.clone(), shutdown.paired_deadline());
980 let mut active: Option<(WorkerRunContext, Option<u64>, Option<u64>, bool)> = None;
981 let outcome = catch_unwind(AssertUnwindSafe(|| {
982 with_worker_info(info, || {
983 if let Err(error) = initializer.initialize(&lifecycle) {
984 send_stream_failure(
985 &results,
986 &shutdown,
987 info.id,
988 initialization_generation,
989 None,
990 None,
991 false,
992 StreamFailure::WorkerInit(error),
993 );
994 return true;
995 }
996 let mut transform = match transform_factory.create(Some(&lifecycle)) {
997 Ok(transform) => transform,
998 Err(error) => {
999 send_stream_failure(
1000 &results,
1001 &shutdown,
1002 info.id,
1003 initialization_generation,
1004 None,
1005 None,
1006 false,
1007 StreamFailure::TransformInit(error),
1008 );
1009 return true;
1010 }
1011 };
1012
1013 loop {
1014 let run = select! {
1015 recv(controls) -> control => match control {
1016 Ok(StreamControl::Begin(run)) => run,
1017 Err(_) => return false,
1018 },
1019 recv(shutdown.signal()) -> _ => return false,
1020 };
1021 active = Some((run.clone(), None, None, false));
1022 run_stream_generation::<S, F, I, M>(
1023 info,
1024 &source_factory,
1025 &mut transform,
1026 &credits,
1027 &results,
1028 &shutdown,
1029 &run,
1030 &mut active,
1031 footprint,
1032 ordered,
1033 );
1034 active = None;
1035 if shutdown.is_cancelled() {
1036 return false;
1037 }
1038 }
1039 })
1040 }));
1041 let fatal = match outcome {
1042 Ok(fatal) => fatal,
1043 Err(_) => {
1044 let (generation, sequence, logical_id, holds_credit) = active
1045 .as_ref()
1046 .map(|(run, sequence, logical_id, held)| {
1047 (run.generation, *sequence, *logical_id, *held)
1048 })
1049 .unwrap_or((initialization_generation, None, None, false));
1050 send_stream_failure(
1051 &results,
1052 &shutdown,
1053 info.id,
1054 generation,
1055 sequence,
1056 logical_id,
1057 holds_credit,
1058 StreamFailure::Panic,
1059 );
1060 true
1061 }
1062 };
1063 if fatal {
1064 shutdown.wait_cancelled();
1065 }
1066}
1067
1068#[allow(clippy::too_many_arguments)]
1069fn run_stream_generation<S, F, I, M>(
1070 info: WorkerInfo,
1071 source_factory: &S,
1072 transform: &mut F::Transform,
1073 credits: &Receiver<()>,
1074 results: &Sender<StreamCompletionFor<S, F, I, M>>,
1075 shutdown: &CancellationToken,
1076 run: &WorkerRunContext,
1077 active: &mut Option<(WorkerRunContext, Option<u64>, Option<u64>, bool)>,
1078 footprint: Option<StreamFootprint<S, F>>,
1079 ordered: bool,
1080) where
1081 S: WorkerSourceFactory,
1082 F: TransformFactory<S::Sample> + Send + Sync + 'static,
1083 F::Transform: Send + 'static,
1084 TransformOutput<S, F>: Send + 'static,
1085 TransformFailure<S, F>: Send + 'static,
1086 F::Error: Send + 'static,
1087 I: WorkerInit + Send + Sync + 'static,
1088 I::Error: Send + 'static,
1089 M: StreamMemoryPolicy,
1090{
1091 let generation_info = WorkerInfo::from_loader_seed(
1092 info.id,
1093 info.num_workers,
1094 run.loader_seed,
1095 info.rank,
1096 run.generation,
1097 )
1098 .expect("validated stream worker identity remains valid for every generation");
1099 let context = WorkerContext::new(
1100 generation_info,
1101 run.cancellation.clone(),
1102 run.deadline.clone(),
1103 );
1104 let mut source = match source_factory.create(context.clone()) {
1105 Ok(source) if context.check().is_ok() => source,
1106 Ok(source) => {
1107 drop(source);
1108 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, false);
1109 return;
1110 }
1111 Err(error) => {
1112 send_stream_failure(
1113 results,
1114 shutdown,
1115 info.id,
1116 run.generation,
1117 None,
1118 None,
1119 false,
1120 StreamFailure::Source(error),
1121 );
1122 run.cancel();
1123 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, false);
1124 return;
1125 }
1126 };
1127
1128 let mut last_sequence = None;
1129 loop {
1130 let acquired = select! {
1131 recv(credits) -> credit => credit.is_ok(),
1132 recv(run.cancellation.signal()) -> _ => false,
1133 recv(shutdown.signal()) -> _ => return,
1134 };
1135 if !acquired {
1136 drop(source);
1137 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, false);
1138 return;
1139 }
1140 active.as_mut().expect("active stream generation").3 = true;
1141 if context.check().is_err() {
1142 drop(source);
1143 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1144 active.as_mut().expect("active stream generation").3 = false;
1145 return;
1146 }
1147
1148 let record = match source.next() {
1149 Some(Ok(record)) => {
1150 if context.check().is_err() {
1151 drop(source);
1152 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1153 active.as_mut().expect("active stream generation").3 = false;
1154 return;
1155 }
1156 record
1157 }
1158 None => {
1159 drop(source);
1160 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1161 active.as_mut().expect("active stream generation").3 = false;
1162 return;
1163 }
1164 Some(Err(error)) => {
1165 drop(source);
1166 send_stream_failure(
1167 results,
1168 shutdown,
1169 info.id,
1170 run.generation,
1171 None,
1172 None,
1173 true,
1174 StreamFailure::Source(error),
1175 );
1176 active.as_mut().expect("active stream generation").3 = false;
1177 run.cancel();
1178 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, false);
1179 return;
1180 }
1181 };
1182 let sequence = record.sequence.map(SequenceId::get);
1183 let logical_id = record.logical_id.get();
1184 {
1185 let active = active.as_mut().expect("active stream generation");
1186 active.1 = sequence;
1187 active.2 = Some(logical_id);
1188 }
1189 let task_context = TaskContext {
1190 loader_seed: run.loader_seed,
1191 epoch: run.epoch,
1192 rank: info.rank,
1193 logical_sample: logical_id,
1194 stage: 0,
1195 cancellation: run.cancellation.clone(),
1196 deadline: run.deadline.clone(),
1197 };
1198 let sample = match transform.transform(record.sample, &task_context) {
1199 Ok(sample) if context.check().is_ok() => sample,
1200 Ok(_) => {
1201 drop(source);
1202 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1203 active.as_mut().expect("active stream generation").3 = false;
1204 return;
1205 }
1206 Err(error) => {
1207 drop(source);
1208 send_stream_failure(
1209 results,
1210 shutdown,
1211 info.id,
1212 run.generation,
1213 sequence,
1214 Some(logical_id),
1215 true,
1216 StreamFailure::Transform(error),
1217 );
1218 active.as_mut().expect("active stream generation").3 = false;
1219 run.cancel();
1220 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, false);
1221 return;
1222 }
1223 };
1224 let permit = if let Some(footprint) = footprint {
1225 let actual = footprint(&sample);
1226 let budget = run
1227 .byte_budget
1228 .as_ref()
1229 .expect("enabled stream byte accounting has a generation budget");
1230 let acquired = if ordered {
1231 let Some(sequence) = sequence else {
1232 drop(source);
1233 send_stream_failure(
1234 results,
1235 shutdown,
1236 info.id,
1237 run.generation,
1238 None,
1239 Some(logical_id),
1240 true,
1241 StreamFailure::Memory(
1242 M::protocol(
1243 None,
1244 "ordered byte-bounded stream record did not provide a sequence identifier".to_owned(),
1245 )
1246 .expect("enabled stream accounting has protocol failure storage"),
1247 ),
1248 );
1249 active.as_mut().expect("active stream generation").3 = false;
1250 run.cancel();
1251 send_stream_end::<S, F, I, M>(
1252 results,
1253 shutdown,
1254 info.id,
1255 run.generation,
1256 false,
1257 );
1258 return;
1259 };
1260 if last_sequence.is_some_and(|previous| sequence <= previous) {
1261 drop(source);
1262 send_stream_failure(
1263 results,
1264 shutdown,
1265 info.id,
1266 run.generation,
1267 Some(sequence),
1268 Some(logical_id),
1269 true,
1270 StreamFailure::Memory(
1271 M::protocol(
1272 Some(sequence),
1273 format!(
1274 "worker {} emitted non-increasing sequence {sequence}",
1275 info.id
1276 ),
1277 )
1278 .expect("enabled stream accounting has protocol failure storage"),
1279 ),
1280 );
1281 active.as_mut().expect("active stream generation").3 = false;
1282 run.cancel();
1283 send_stream_end::<S, F, I, M>(
1284 results,
1285 shutdown,
1286 info.id,
1287 run.generation,
1288 false,
1289 );
1290 return;
1291 }
1292 last_sequence = Some(sequence);
1293 if !send_stream_completion(
1294 results,
1295 shutdown,
1296 Some(&run.cancellation),
1297 StreamCompletion {
1298 worker: info.id,
1299 generation: run.generation,
1300 sequence: Some(sequence),
1301 logical_id: Some(logical_id),
1302 holds_credit: false,
1303 result: Ok(StreamMessage::Waiting(
1304 M::waiting(sequence)
1305 .expect("enabled stream accounting has waiter messages"),
1306 )),
1307 },
1308 ) {
1309 drop(source);
1310 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1311 active.as_mut().expect("active stream generation").3 = false;
1312 return;
1313 }
1314 budget.acquire_ordered(sequence, actual)
1315 } else {
1316 budget.acquire(actual)
1317 };
1318 match acquired {
1319 Ok(permit) => Some(permit),
1320 Err(BudgetError::Cancelled) => {
1321 drop(source);
1322 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1323 active.as_mut().expect("active stream generation").3 = false;
1324 return;
1325 }
1326 Err(BudgetError::Oversize { limit, actual }) => {
1327 drop(source);
1328 send_stream_failure(
1329 results,
1330 shutdown,
1331 info.id,
1332 run.generation,
1333 sequence,
1334 Some(logical_id),
1335 true,
1336 StreamFailure::Memory(
1337 M::memory_limit(limit, actual)
1338 .expect("enabled stream accounting has memory-limit storage"),
1339 ),
1340 );
1341 active.as_mut().expect("active stream generation").3 = false;
1342 run.cancel();
1343 send_stream_end::<S, F, I, M>(
1344 results,
1345 shutdown,
1346 info.id,
1347 run.generation,
1348 false,
1349 );
1350 return;
1351 }
1352 Err(BudgetError::SequenceAlreadyAdmitted) => {
1353 drop(source);
1354 send_stream_failure(
1355 results,
1356 shutdown,
1357 info.id,
1358 run.generation,
1359 sequence,
1360 Some(logical_id),
1361 true,
1362 StreamFailure::Memory(
1363 M::protocol(
1364 sequence,
1365 "sequence was already admitted to the byte budget".to_owned(),
1366 )
1367 .expect("enabled stream accounting has protocol failure storage"),
1368 ),
1369 );
1370 active.as_mut().expect("active stream generation").3 = false;
1371 run.cancel();
1372 send_stream_end::<S, F, I, M>(
1373 results,
1374 shutdown,
1375 info.id,
1376 run.generation,
1377 false,
1378 );
1379 return;
1380 }
1381 }
1382 } else {
1383 None
1384 };
1385 let sent = send_stream_completion(
1386 results,
1387 shutdown,
1388 Some(&run.cancellation),
1389 StreamCompletion {
1390 worker: info.id,
1391 generation: run.generation,
1392 sequence,
1393 logical_id: Some(logical_id),
1394 holds_credit: true,
1395 result: Ok(StreamMessage::Record(
1396 WorkerRecord {
1397 sequence: record.sequence,
1398 logical_id: record.logical_id,
1399 sample,
1400 },
1401 M::permit(permit),
1402 )),
1403 },
1404 );
1405 if sent {
1406 let active = active.as_mut().expect("active stream generation");
1407 active.1 = None;
1408 active.2 = None;
1409 active.3 = false;
1410 } else {
1411 drop(source);
1412 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1413 active.as_mut().expect("active stream generation").3 = false;
1414 return;
1415 }
1416 }
1417}
1418
1419fn send_stream_end<S, F, I, M>(
1420 results: &Sender<StreamCompletionFor<S, F, I, M>>,
1421 shutdown: &CancellationToken,
1422 worker: usize,
1423 generation: u64,
1424 holds_credit: bool,
1425) where
1426 S: WorkerSourceFactory,
1427 F: TransformFactory<S::Sample>,
1428 I: WorkerInit,
1429 M: StreamMemoryPolicy,
1430{
1431 let _ = send_stream_completion(
1432 results,
1433 shutdown,
1434 None,
1435 StreamCompletion {
1436 worker,
1437 generation,
1438 sequence: None,
1439 logical_id: None,
1440 holds_credit,
1441 result: Ok(StreamMessage::End),
1442 },
1443 );
1444}
1445
1446#[allow(clippy::too_many_arguments)]
1447fn send_stream_failure<T, P, W, ME, SE, TE, FE, IE>(
1448 results: &StreamCompletionSender<T, P, W, ME, SE, TE, FE, IE>,
1449 shutdown: &CancellationToken,
1450 worker: usize,
1451 generation: u64,
1452 sequence: Option<u64>,
1453 logical_id: Option<u64>,
1454 holds_credit: bool,
1455 failure: StreamFailure<SE, TE, FE, IE, ME>,
1456) {
1457 let _ = send_stream_completion(
1458 results,
1459 shutdown,
1460 None,
1461 StreamCompletion {
1462 worker,
1463 generation,
1464 sequence,
1465 logical_id,
1466 holds_credit,
1467 result: Err(failure),
1468 },
1469 );
1470}
1471
1472fn send_stream_completion<T, P, W, ME, SE, TE, FE, IE>(
1473 results: &StreamCompletionSender<T, P, W, ME, SE, TE, FE, IE>,
1474 shutdown: &CancellationToken,
1475 cancellation: Option<&CancellationToken>,
1476 completion: StreamCompletion<T, P, W, ME, SE, TE, FE, IE>,
1477) -> bool {
1478 if let Some(cancellation) = cancellation {
1479 select! {
1480 send(results, completion) -> result => result.is_ok(),
1481 recv(cancellation.signal()) -> _ => false,
1482 recv(shutdown.signal()) -> _ => false,
1483 }
1484 } else {
1485 select! {
1486 send(results, completion) -> result => result.is_ok(),
1487 recv(shutdown.signal()) -> _ => false,
1488 }
1489 }
1490}
1491
1492#[derive(Clone, Copy)]
1493struct StreamConfiguration {
1494 workers: usize,
1495 batch_size: usize,
1496 drop_last: bool,
1497 prefetch_factor: usize,
1498 ordered: bool,
1499 persistent_workers: bool,
1500 timeout: Option<Duration>,
1501 loader_seed: u64,
1502 epoch: u64,
1503 rank: usize,
1504 prefetch_bytes: Option<NonZeroUsize>,
1505 pin_memory: StreamPinRequest,
1506}
1507
1508#[derive(Clone, Copy)]
1509enum StreamPinRequest {
1510 Disabled,
1511 Auto,
1512 Explicit(Device),
1513}
1514
1515impl Default for StreamConfiguration {
1516 fn default() -> Self {
1517 Self {
1518 workers: 1,
1519 batch_size: 1,
1520 drop_last: false,
1521 prefetch_factor: 2,
1522 ordered: true,
1523 persistent_workers: false,
1524 timeout: None,
1525 loader_seed: 0,
1526 epoch: 0,
1527 rank: 0,
1528 prefetch_bytes: None,
1529 pin_memory: StreamPinRequest::Disabled,
1530 }
1531 }
1532}
1533
1534pub struct StreamDataLoaderBuilder<
1536 S,
1537 C = DefaultCollator,
1538 F = IdentityTransformFactory,
1539 I = NoWorkerInit,
1540 M = MemoryDisabled,
1541 N = PinDisabled,
1542 K = crate::CheckpointDisabled,
1543> {
1544 factory: S,
1545 collator: C,
1546 transform_factory: F,
1547 worker_init: I,
1548 configuration: StreamConfiguration,
1549 checkpoint: K,
1550 states: PhantomData<(M, N)>,
1551}
1552
1553impl<S> StreamDataLoaderBuilder<S>
1554where
1555 S: WorkerSourceFactory,
1556{
1557 pub fn new(factory: S) -> Self {
1560 Self {
1561 factory,
1562 collator: DefaultCollator,
1563 transform_factory: IdentityTransformFactory,
1564 worker_init: NoWorkerInit,
1565 configuration: StreamConfiguration::default(),
1566 checkpoint: crate::CheckpointDisabled,
1567 states: PhantomData,
1568 }
1569 }
1570}
1571
1572impl<S, C, F, I, M, N, K> StreamDataLoaderBuilder<S, C, F, I, M, N, K> {
1573 pub fn workers(mut self, workers: usize) -> Self {
1575 self.configuration.workers = workers;
1576 self
1577 }
1578
1579 pub fn batch_size(mut self, batch_size: usize) -> Self {
1581 self.configuration.batch_size = batch_size;
1582 self
1583 }
1584
1585 pub fn drop_last(mut self, drop_last: bool) -> Self {
1587 self.configuration.drop_last = drop_last;
1588 self
1589 }
1590
1591 pub fn prefetch_factor(mut self, factor: usize) -> Self {
1596 self.configuration.prefetch_factor = factor;
1597 self
1598 }
1599
1600 pub fn prefetch_bytes(
1606 mut self,
1607 limit: NonZeroUsize,
1608 ) -> StreamDataLoaderBuilder<S, C, F, I, MemoryEnabled, N, K> {
1609 self.configuration.prefetch_bytes = Some(limit);
1610 StreamDataLoaderBuilder {
1611 factory: self.factory,
1612 collator: self.collator,
1613 transform_factory: self.transform_factory,
1614 worker_init: self.worker_init,
1615 configuration: self.configuration,
1616 checkpoint: self.checkpoint,
1617 states: PhantomData,
1618 }
1619 }
1620
1621 pub fn pin_memory(mut self) -> StreamDataLoaderBuilder<S, C, F, I, M, PinEnabled<Auto>, K> {
1623 self.configuration.pin_memory = StreamPinRequest::Auto;
1624 StreamDataLoaderBuilder {
1625 factory: self.factory,
1626 collator: self.collator,
1627 transform_factory: self.transform_factory,
1628 worker_init: self.worker_init,
1629 configuration: self.configuration,
1630 checkpoint: self.checkpoint,
1631 states: PhantomData,
1632 }
1633 }
1634
1635 pub fn pin_memory_for(
1637 mut self,
1638 device: Device,
1639 ) -> StreamDataLoaderBuilder<S, C, F, I, M, PinEnabled<Explicit>, K> {
1640 self.configuration.pin_memory = StreamPinRequest::Explicit(device);
1641 StreamDataLoaderBuilder {
1642 factory: self.factory,
1643 collator: self.collator,
1644 transform_factory: self.transform_factory,
1645 worker_init: self.worker_init,
1646 configuration: self.configuration,
1647 checkpoint: self.checkpoint,
1648 states: PhantomData,
1649 }
1650 }
1651
1652 pub fn ordered(mut self, ordered: bool) -> Self {
1654 self.configuration.ordered = ordered;
1655 self
1656 }
1657
1658 pub fn in_order(self, in_order: bool) -> Self {
1660 self.ordered(in_order)
1661 }
1662
1663 pub fn seed(mut self, seed: u64) -> Self {
1665 self.configuration.loader_seed = seed;
1666 self
1667 }
1668
1669 pub fn epoch(mut self, epoch: u64) -> Self {
1671 self.configuration.epoch = epoch;
1672 self
1673 }
1674
1675 pub fn rank(mut self, rank: usize) -> Self {
1677 self.configuration.rank = rank;
1678 self
1679 }
1680
1681 pub fn timeout(mut self, timeout: Duration) -> Self {
1683 self.configuration.timeout = (!timeout.is_zero()).then_some(timeout);
1684 self
1685 }
1686
1687 pub fn persistent_workers(mut self, persistent: bool) -> Self {
1689 self.configuration.persistent_workers = persistent;
1690 self
1691 }
1692
1693 pub fn collate<C2>(self, collator: C2) -> StreamDataLoaderBuilder<S, C2, F, I, M, N, K> {
1695 StreamDataLoaderBuilder {
1696 factory: self.factory,
1697 collator,
1698 transform_factory: self.transform_factory,
1699 worker_init: self.worker_init,
1700 configuration: self.configuration,
1701 checkpoint: self.checkpoint,
1702 states: PhantomData,
1703 }
1704 }
1705
1706 pub fn transform<T>(
1708 self,
1709 transform: T,
1710 ) -> StreamDataLoaderBuilder<S, C, CloneTransformFactory<T>, I, M, N, K> {
1711 StreamDataLoaderBuilder {
1712 factory: self.factory,
1713 collator: self.collator,
1714 transform_factory: CloneTransformFactory::new(transform),
1715 worker_init: self.worker_init,
1716 configuration: self.configuration,
1717 checkpoint: self.checkpoint,
1718 states: PhantomData,
1719 }
1720 }
1721
1722 pub fn transform_factory<F2>(
1724 self,
1725 factory: F2,
1726 ) -> StreamDataLoaderBuilder<S, C, F2, I, M, N, K> {
1727 StreamDataLoaderBuilder {
1728 factory: self.factory,
1729 collator: self.collator,
1730 transform_factory: factory,
1731 worker_init: self.worker_init,
1732 configuration: self.configuration,
1733 checkpoint: self.checkpoint,
1734 states: PhantomData,
1735 }
1736 }
1737
1738 pub fn worker_init<I2>(self, worker_init: I2) -> StreamDataLoaderBuilder<S, C, F, I2, M, N, K> {
1740 StreamDataLoaderBuilder {
1741 factory: self.factory,
1742 collator: self.collator,
1743 transform_factory: self.transform_factory,
1744 worker_init,
1745 configuration: self.configuration,
1746 checkpoint: self.checkpoint,
1747 states: PhantomData,
1748 }
1749 }
1750
1751 fn build_inner(
1753 self,
1754 footprint: Option<StreamFootprint<S, F>>,
1755 ) -> Result<StreamDataLoader<S, C, F, I, M, N>>
1756 where
1757 S: WorkerSourceFactory,
1758 F: TransformFactory<S::Sample>,
1759 C: Collate<TransformOutput<S, F>>,
1760 I: WorkerInit,
1761 M: StreamMemoryPolicy,
1762 {
1763 let pin_memory_status = resolve_stream_pin(self.configuration.pin_memory)?;
1764 if self.configuration.workers == 0 {
1765 return Err(invalid_configuration(
1766 "workers",
1767 "stream workers must be positive; use batches or batches_with_collate for an ordinary iterator",
1768 ));
1769 }
1770 if self.configuration.batch_size == 0 {
1771 return Err(invalid_configuration(
1772 "batch_size",
1773 "must be greater than zero",
1774 ));
1775 }
1776 if self.configuration.prefetch_factor == 0 {
1777 return Err(invalid_configuration(
1778 "prefetch_factor",
1779 "must be greater than zero",
1780 ));
1781 }
1782 if self
1783 .configuration
1784 .timeout
1785 .is_some_and(|timeout| !Deadline::can_represent(timeout))
1786 {
1787 return Err(invalid_configuration(
1788 "timeout",
1789 "exceeds the platform monotonic clock range",
1790 ));
1791 }
1792 let outstanding_capacity = self
1793 .configuration
1794 .workers
1795 .checked_mul(self.configuration.prefetch_factor)
1796 .ok_or_else(|| {
1797 invalid_configuration(
1798 "prefetch_factor",
1799 "workers multiplied by prefetch_factor exceeds usize",
1800 )
1801 })?;
1802 validate_stream_capacity::<S, F, I, M>(
1803 self.configuration.workers,
1804 self.configuration.prefetch_factor,
1805 outstanding_capacity,
1806 self.configuration.batch_size,
1807 usize::from(self.configuration.ordered) * outstanding_capacity,
1808 )?;
1809 let mut ordered_reassembly = Vec::new();
1810 if self.configuration.ordered {
1811 ordered_reassembly
1812 .try_reserve_exact(outstanding_capacity)
1813 .map_err(|error| {
1814 capacity_error(format!(
1815 "ordered stream reassembly capacity is unavailable: {error}"
1816 ))
1817 })?;
1818 }
1819 validate_stream_capacity::<S, F, I, M>(
1820 self.configuration.workers,
1821 self.configuration.prefetch_factor,
1822 outstanding_capacity,
1823 self.configuration.batch_size,
1824 ordered_reassembly.capacity(),
1825 )?;
1826 let exact_len = self.factory.exact_len();
1827 Ok(StreamDataLoader {
1828 factory: Arc::new(self.factory),
1829 collator: self.collator,
1830 transform_factory: Arc::new(self.transform_factory),
1831 worker_init: Arc::new(self.worker_init),
1832 configuration: self.configuration,
1833 outstanding_capacity,
1834 exact_len,
1835 next_generation: 0,
1836 persistent_pool: None,
1837 ordered_reassembly: Mutex::new(ordered_reassembly),
1838 footprint,
1839 pin_memory_status,
1840 policies: PhantomData,
1841 })
1842 }
1843}
1844
1845fn stream_resident_bytes<T: MemoryFootprint>(value: &T) -> usize {
1846 value.resident_bytes()
1847}
1848
1849fn resolve_stream_pin(request: StreamPinRequest) -> Result<PinMemoryStatus> {
1850 match request {
1851 StreamPinRequest::Disabled => Ok(PinMemoryStatus::Disabled),
1852 StreamPinRequest::Auto => {
1853 let capabilities = available_devices();
1854 if capabilities.cuda && capabilities.cuda_device_count > 0 {
1855 Ok(PinMemoryStatus::Enabled(Device::Cuda(0)))
1856 } else {
1857 Ok(PinMemoryStatus::DisabledNoAccelerator)
1858 }
1859 }
1860 StreamPinRequest::Explicit(Device::Cuda(index)) => {
1861 let capabilities = available_devices();
1862 if capabilities.cuda && index < capabilities.cuda_device_count {
1863 Ok(PinMemoryStatus::Enabled(Device::Cuda(index)))
1864 } else {
1865 Err(invalid_configuration(
1866 "pin_memory",
1867 format!(
1868 "CUDA device {index} was requested, but the linked runtime exposes {} available CUDA device(s)",
1869 capabilities.cuda_device_count
1870 ),
1871 ))
1872 }
1873 }
1874 StreamPinRequest::Explicit(device) => Err(invalid_configuration(
1875 "pin_memory",
1876 format!("only an available CUDA device can back pinned host memory, got {device:?}"),
1877 )),
1878 }
1879}
1880
1881impl<S, C, F, I> StreamDataLoaderBuilder<S, C, F, I, MemoryDisabled, PinDisabled> {
1882 pub fn build(self) -> Result<StreamDataLoader<S, C, F, I, MemoryDisabled, PinDisabled>>
1884 where
1885 S: WorkerSourceFactory,
1886 F: TransformFactory<S::Sample>,
1887 C: Collate<TransformOutput<S, F>>,
1888 I: WorkerInit,
1889 {
1890 self.build_inner(None)
1891 }
1892}
1893
1894impl<S, C, F, I> StreamDataLoaderBuilder<S, C, F, I, MemoryEnabled, PinDisabled> {
1895 pub fn build(self) -> Result<StreamDataLoader<S, C, F, I, MemoryEnabled, PinDisabled>>
1897 where
1898 S: WorkerSourceFactory,
1899 F: TransformFactory<S::Sample>,
1900 TransformOutput<S, F>: MemoryFootprint,
1901 C: Collate<TransformOutput<S, F>>,
1902 I: WorkerInit,
1903 {
1904 self.build_inner(Some(stream_resident_bytes::<TransformOutput<S, F>>))
1905 }
1906}
1907
1908impl<S, C, F, I, Q> StreamDataLoaderBuilder<S, C, F, I, MemoryDisabled, PinEnabled<Q>> {
1909 pub fn build(self) -> Result<StreamDataLoader<S, C, F, I, MemoryDisabled, PinEnabled<Q>>>
1911 where
1912 S: WorkerSourceFactory,
1913 F: TransformFactory<S::Sample>,
1914 C: Collate<TransformOutput<S, F>>,
1915 C::Batch: PinMemory,
1916 I: WorkerInit,
1917 {
1918 self.build_inner(None)
1919 }
1920}
1921
1922impl<S, C, F, I, Q> StreamDataLoaderBuilder<S, C, F, I, MemoryEnabled, PinEnabled<Q>> {
1923 pub fn build(self) -> Result<StreamDataLoader<S, C, F, I, MemoryEnabled, PinEnabled<Q>>>
1925 where
1926 S: WorkerSourceFactory,
1927 F: TransformFactory<S::Sample>,
1928 TransformOutput<S, F>: MemoryFootprint,
1929 C: Collate<TransformOutput<S, F>>,
1930 C::Batch: PinMemory,
1931 I: WorkerInit,
1932 {
1933 self.build_inner(Some(stream_resident_bytes::<TransformOutput<S, F>>))
1934 }
1935}
1936
1937#[allow(private_bounds)]
1939pub struct StreamDataLoader<
1940 S,
1941 C,
1942 F = IdentityTransformFactory,
1943 I = NoWorkerInit,
1944 M = MemoryDisabled,
1945 N = PinDisabled,
1946> where
1947 S: WorkerSourceFactory,
1948 F: TransformFactory<S::Sample>,
1949 C: Collate<TransformOutput<S, F>>,
1950 I: WorkerInit,
1951 M: StreamMemoryPolicy,
1952{
1953 factory: Arc<S>,
1954 collator: C,
1955 transform_factory: Arc<F>,
1956 worker_init: Arc<I>,
1957 configuration: StreamConfiguration,
1958 outstanding_capacity: usize,
1959 exact_len: Option<usize>,
1960 next_generation: u64,
1961 persistent_pool: Option<StreamWorkerPool<S, F, I, M>>,
1962 ordered_reassembly: RetainedReassembly<TransformOutput<S, F>, M::Permit>,
1965 footprint: Option<StreamFootprint<S, F>>,
1966 pin_memory_status: PinMemoryStatus,
1967 policies: PhantomData<(M, N)>,
1968}
1969
1970#[allow(private_bounds)]
1971impl<S, C, F, I, M, N> StreamDataLoader<S, C, F, I, M, N>
1972where
1973 S: WorkerSourceFactory,
1974 F: TransformFactory<S::Sample>,
1975 C: Collate<TransformOutput<S, F>>,
1976 I: WorkerInit,
1977 M: StreamMemoryPolicy,
1978{
1979 pub fn len(&self) -> Option<usize> {
1981 self.exact_len.map(|length| {
1982 let complete = length / self.configuration.batch_size;
1983 complete
1984 + usize::from(
1985 !self.configuration.drop_last
1986 && !length.is_multiple_of(self.configuration.batch_size),
1987 )
1988 })
1989 }
1990
1991 pub fn is_empty(&self) -> Option<bool> {
1993 self.len().map(|length| length == 0)
1994 }
1995
1996 pub fn workers(&self) -> usize {
1998 self.configuration.workers
1999 }
2000
2001 pub fn effective_prefetch_factor(&self) -> NonZeroUsize {
2003 NonZeroUsize::new(self.configuration.prefetch_factor)
2004 .expect("validated prefetch factor is nonzero")
2005 }
2006
2007 pub fn effective_prefetch_bytes(&self) -> Option<NonZeroUsize> {
2009 self.configuration.prefetch_bytes
2010 }
2011
2012 pub fn pin_memory_enabled(&self) -> bool {
2014 !matches!(self.pin_memory_status, PinMemoryStatus::Disabled)
2015 }
2016
2017 pub fn pin_memory_status(&self) -> PinMemoryStatus {
2019 self.pin_memory_status
2020 }
2021
2022 pub fn is_ordered(&self) -> bool {
2024 self.configuration.ordered
2025 }
2026
2027 pub fn epoch(&self) -> u64 {
2029 self.configuration.epoch
2030 }
2031
2032 pub fn set_epoch(&mut self, epoch: u64) {
2034 self.configuration.epoch = epoch;
2035 }
2036}
2037
2038struct BufferedRecord<T, P> {
2039 worker: usize,
2040 record: WorkerRecord<T>,
2041 permit: P,
2042}
2043
2044enum StreamIteratorPool<'a, S, F, I, M>
2045where
2046 S: WorkerSourceFactory,
2047 F: TransformFactory<S::Sample>,
2048 I: WorkerInit,
2049 M: StreamMemoryPolicy,
2050{
2051 Owned(StreamWorkerPool<S, F, I, M>),
2052 Persistent(&'a mut StreamWorkerPool<S, F, I, M>),
2053}
2054
2055impl<S, F, I, M> StreamIteratorPool<'_, S, F, I, M>
2056where
2057 S: WorkerSourceFactory,
2058 F: TransformFactory<S::Sample>,
2059 I: WorkerInit,
2060 M: StreamMemoryPolicy,
2061{
2062 fn pool(&self) -> &StreamWorkerPool<S, F, I, M> {
2063 match self {
2064 Self::Owned(pool) => pool,
2065 Self::Persistent(pool) => pool,
2066 }
2067 }
2068
2069 fn pool_mut(&mut self) -> &mut StreamWorkerPool<S, F, I, M> {
2070 match self {
2071 Self::Owned(pool) => pool,
2072 Self::Persistent(pool) => pool,
2073 }
2074 }
2075
2076 fn is_persistent(&self) -> bool {
2077 matches!(self, Self::Persistent(_))
2078 }
2079}
2080
2081#[allow(private_bounds)]
2082impl<S, C, F, I, M, N> StreamDataLoader<S, C, F, I, M, N>
2083where
2084 S: WorkerSourceFactory,
2085 F: TransformFactory<S::Sample> + Send + Sync + 'static,
2086 F::Transform: Send + 'static,
2087 TransformOutput<S, F>: Send + 'static,
2088 TransformFailure<S, F>: Send + 'static,
2089 F::Error: Send + 'static,
2090 C: Collate<TransformOutput<S, F>>,
2091 I: WorkerInit + Send + Sync + 'static,
2092 I::Error: Send + 'static,
2093 M: StreamMemoryPolicy + Send + 'static,
2094 N: StreamPinPolicy<S, C, F>,
2095{
2096 pub fn iter(&mut self) -> StreamLoaderIter<'_, S, C, F, I, M, N> {
2098 let ordered_reassembly = match self.ordered_reassembly.get_mut() {
2099 Ok(reassembly) => reassembly,
2100 Err(poisoned) => poisoned.into_inner(),
2101 };
2102 ordered_reassembly.clear();
2103 let reassembly_allocation_slots = ordered_reassembly.capacity();
2104 let generation = self.next_generation;
2105 let run_context = WorkerRunContext::new(
2106 generation,
2107 self.configuration.loader_seed,
2108 self.configuration.epoch,
2109 )
2110 .with_byte_budget(
2111 self.configuration
2112 .prefetch_bytes
2113 .map(|limit| ByteBudget::new(limit.get())),
2114 );
2115 let mut pending_error = None;
2116 if let Some(next) = generation.checked_add(1) {
2117 self.next_generation = next;
2118 } else {
2119 pending_error = Some(LoaderError::Configuration(invalid_configuration(
2120 "workers",
2121 "stream iterator generation overflowed",
2122 )));
2123 }
2124
2125 let mut partial = Vec::new();
2126 if partial
2127 .try_reserve_exact(self.configuration.batch_size)
2128 .is_err()
2129 {
2130 pending_error = Some(LoaderError::Configuration(invalid_configuration(
2131 "batch_size",
2132 "stream batch allocation is unavailable",
2133 )));
2134 }
2135
2136 let front_waiters = match M::new_waiters(self.configuration.workers) {
2137 Ok(waiters) => waiters,
2138 Err(error) => {
2139 pending_error = Some(LoaderError::Configuration(error));
2140 M::new_waiters(0).expect("zero waiter state is always allocatable")
2141 }
2142 };
2143
2144 let mut pool = None;
2145 if pending_error.is_none() {
2146 if self.configuration.persistent_workers {
2147 if self.persistent_pool.is_none() {
2148 match StreamWorkerPool::new(
2149 Arc::clone(&self.factory),
2150 Arc::clone(&self.transform_factory),
2151 Arc::clone(&self.worker_init),
2152 self.configuration,
2153 generation,
2154 self.outstanding_capacity,
2155 reassembly_allocation_slots,
2156 self.footprint,
2157 ) {
2158 Ok(created) => self.persistent_pool = Some(created),
2159 Err(error) => pending_error = Some(LoaderError::Configuration(error)),
2160 }
2161 }
2162 if pending_error.is_none() {
2163 let persistent = self
2164 .persistent_pool
2165 .as_mut()
2166 .expect("persistent stream pool was created");
2167 if persistent.start_generation(run_context.clone()).is_err() {
2168 pending_error = Some(LoaderError::ChannelClosed { batch: 0 });
2169 } else {
2170 pool = Some(StreamIteratorPool::Persistent(persistent));
2171 }
2172 }
2173 } else {
2174 match StreamWorkerPool::new(
2175 Arc::clone(&self.factory),
2176 Arc::clone(&self.transform_factory),
2177 Arc::clone(&self.worker_init),
2178 self.configuration,
2179 generation,
2180 self.outstanding_capacity,
2181 reassembly_allocation_slots,
2182 self.footprint,
2183 ) {
2184 Ok(mut created) => {
2185 if created.start_generation(run_context.clone()).is_err() {
2186 pending_error = Some(LoaderError::ChannelClosed { batch: 0 });
2187 } else {
2188 pool = Some(StreamIteratorPool::Owned(created));
2189 }
2190 }
2191 Err(error) => pending_error = Some(LoaderError::Configuration(error)),
2192 }
2193 }
2194 }
2195
2196 StreamLoaderIter {
2197 collator: &mut self.collator,
2198 pool,
2199 run_context,
2200 generation,
2201 ordered: self.configuration.ordered,
2202 timeout: self.configuration.timeout,
2203 batch_size: self.configuration.batch_size,
2204 drop_last: self.configuration.drop_last,
2205 expected_records: self.exact_len,
2206 reassembly_capacity: self.outstanding_capacity,
2207 completed: ordered_reassembly,
2208 partial,
2209 next_sequence: 0,
2210 next_batch: 0,
2211 pending_error,
2212 source_complete: false,
2213 exhausted: false,
2214 front_waiters,
2215 pin_memory_status: self.pin_memory_status,
2216 marker: PhantomData,
2217 }
2218 }
2219}
2220
2221#[allow(private_bounds)]
2223pub struct StreamLoaderIter<'a, S, C, F, I, M = MemoryDisabled, N = PinDisabled>
2224where
2225 S: WorkerSourceFactory,
2226 F: TransformFactory<S::Sample>,
2227 C: Collate<TransformOutput<S, F>>,
2228 I: WorkerInit,
2229 M: StreamMemoryPolicy,
2230{
2231 collator: &'a mut C,
2232 pool: Option<StreamIteratorPool<'a, S, F, I, M>>,
2233 run_context: WorkerRunContext,
2234 generation: u64,
2235 ordered: bool,
2236 timeout: Option<Duration>,
2237 batch_size: usize,
2238 drop_last: bool,
2239 expected_records: Option<usize>,
2240 reassembly_capacity: usize,
2241 completed: &'a mut Vec<ReassemblyEntry<TransformOutput<S, F>, M::Permit>>,
2242 partial: Vec<TransformOutput<S, F>>,
2243 next_sequence: u64,
2244 next_batch: u64,
2245 pending_error: Option<StreamLoaderError<S, C, F, I>>,
2246 source_complete: bool,
2247 exhausted: bool,
2248 front_waiters: M::Waiters,
2249 pin_memory_status: PinMemoryStatus,
2250 marker: PhantomData<&'a mut (S, C, F, I, M, N)>,
2251}
2252
2253#[allow(private_bounds)]
2254impl<S, C, F, I, M, N> StreamLoaderIter<'_, S, C, F, I, M, N>
2255where
2256 S: WorkerSourceFactory,
2257 F: TransformFactory<S::Sample> + Send + Sync + 'static,
2258 F::Transform: Send + 'static,
2259 TransformOutput<S, F>: Send + 'static,
2260 TransformFailure<S, F>: Send + 'static,
2261 F::Error: Send + 'static,
2262 C: Collate<TransformOutput<S, F>>,
2263 I: WorkerInit + Send + Sync + 'static,
2264 I::Error: Send + 'static,
2265 M: StreamMemoryPolicy + Send + 'static,
2266 N: StreamPinPolicy<S, C, F>,
2267{
2268 fn return_credit(&mut self, worker: usize) -> std::result::Result<(), ()> {
2269 self.pool.as_ref().ok_or(())?.pool().return_credit(worker)
2270 }
2271
2272 fn disarm_deadline(&self) {
2273 self.run_context.deadline.disarm();
2274 }
2275
2276 fn set_front_waiter(&mut self, worker: usize, sequence: Option<u64>) {
2277 if let Some(waiters) = M::waiters_mut(&mut self.front_waiters) {
2278 waiters[worker] = sequence;
2279 }
2280 }
2281
2282 fn close(&mut self, poisoned: bool) {
2283 self.exhausted = true;
2284 self.run_context.cancel();
2285 let mut credit_failed = false;
2286 while let Some((_, buffered)) = self.completed.pop() {
2287 credit_failed |= self.return_credit(buffered.worker).is_err();
2288 }
2289 if let Some(mut pool) = self.pool.take() {
2290 let persistent = pool.is_persistent();
2291 let quiesce_failed = persistent && pool.pool_mut().quiesce(self.generation).is_err();
2292 if poisoned || credit_failed || quiesce_failed {
2293 pool.pool_mut().poison();
2294 }
2295 }
2296 self.partial.clear();
2297 }
2298
2299 fn protocol_error(
2300 &mut self,
2301 sequence: Option<u64>,
2302 reason: impl Into<String>,
2303 ) -> StreamLoaderError<S, C, F, I> {
2304 self.disarm_deadline();
2305 self.close(false);
2306 LoaderError::StreamProtocol {
2307 sequence,
2308 reason: reason.into(),
2309 }
2310 }
2311
2312 fn consume(
2313 &mut self,
2314 buffered: BufferedRecord<TransformOutput<S, F>, M::Permit>,
2315 ) -> std::result::Result<bool, StreamLoaderError<S, C, F, I>> {
2316 let BufferedRecord {
2317 worker,
2318 record,
2319 permit,
2320 } = buffered;
2321 let next_sequence = if self.ordered {
2322 let Some(next) = record
2323 .sequence
2324 .expect("ordered records are validated")
2325 .checked_next()
2326 else {
2327 self.run_context.cancel();
2328 if self.return_credit(worker).is_err() {
2329 self.close(true);
2330 return Err(LoaderError::ChannelClosed {
2331 batch: self.next_batch,
2332 });
2333 }
2334 return Err(self.protocol_error(
2335 Some(self.next_sequence),
2336 "stream sequence identifier overflowed",
2337 ));
2338 };
2339 Some(next.get())
2340 } else {
2341 None
2342 };
2343 if self.return_credit(worker).is_err() {
2344 self.close(true);
2345 return Err(LoaderError::ChannelClosed {
2346 batch: self.next_batch,
2347 });
2348 }
2349 if let Some(next_sequence) = next_sequence {
2350 self.next_sequence = next_sequence;
2351 }
2352 drop(permit);
2353 self.partial.push(record.sample);
2354 Ok(self.partial.len() == self.batch_size)
2355 }
2356
2357 fn finish_batch(&mut self) -> std::result::Result<C::Batch, StreamLoaderError<S, C, F, I>> {
2358 let mut replacement = Vec::new();
2359 if replacement.try_reserve_exact(self.batch_size).is_err() {
2360 self.close(false);
2361 return Err(LoaderError::Configuration(invalid_configuration(
2362 "batch_size",
2363 "stream batch allocation is unavailable",
2364 )));
2365 }
2366 let samples = std::mem::replace(&mut self.partial, replacement);
2367 let batch = self.next_batch;
2368 let result = match catch_unwind(AssertUnwindSafe(|| self.collator.collate(samples))) {
2369 Ok(result) => result.map_err(|source| LoaderError::Pipeline {
2370 batch: Some(batch),
2371 worker: None,
2372 source: PipelineError::Collate(source),
2373 }),
2374 Err(_) => Err(LoaderError::CoordinatorPanic {
2375 stage: "stream collation",
2376 batch: Some(batch),
2377 }),
2378 }
2379 .and_then(|value| match self.pin_memory_status {
2380 PinMemoryStatus::Enabled(device) => {
2381 N::pin(value, device).map_err(|source| LoaderError::PinMemory { batch, source })
2382 }
2383 PinMemoryStatus::Disabled | PinMemoryStatus::DisabledNoAccelerator => Ok(value),
2384 });
2385 match result {
2386 Ok(batch) => match self.next_batch.checked_add(1) {
2387 Some(next) => {
2388 self.next_batch = next;
2389 self.disarm_deadline();
2390 Ok(batch)
2391 }
2392 None => {
2393 self.close(false);
2394 Err(LoaderError::Configuration(invalid_configuration(
2395 "batch_size",
2396 "stream batch sequence overflowed",
2397 )))
2398 }
2399 },
2400 Err(error) => {
2401 self.disarm_deadline();
2402 self.close(false);
2403 Err(error)
2404 }
2405 }
2406 }
2407
2408 fn map_failure(
2409 &mut self,
2410 worker: usize,
2411 sequence: Option<u64>,
2412 logical_id: Option<u64>,
2413 failure: StreamFailureFor<S, F, I, M>,
2414 ) -> StreamLoaderError<S, C, F, I> {
2415 let batch = Some(self.next_batch);
2416 match failure {
2417 StreamFailure::Source(source) => LoaderError::StreamPipeline {
2418 batch,
2419 worker,
2420 sequence,
2421 logical_id,
2422 source: PipelineError::Source(source),
2423 },
2424 StreamFailure::Transform(source) => LoaderError::StreamPipeline {
2425 batch,
2426 worker,
2427 sequence,
2428 logical_id,
2429 source: PipelineError::Transform(source),
2430 },
2431 StreamFailure::TransformInit(source) => LoaderError::StreamPipeline {
2432 batch: None,
2433 worker,
2434 sequence: None,
2435 logical_id: None,
2436 source: PipelineError::TransformInit(source),
2437 },
2438 StreamFailure::WorkerInit(source) => LoaderError::StreamPipeline {
2439 batch: None,
2440 worker,
2441 sequence: None,
2442 logical_id: None,
2443 source: PipelineError::WorkerInit(source),
2444 },
2445 StreamFailure::Memory(failure) => M::map_failure(failure, worker, sequence, logical_id),
2446 StreamFailure::Panic => LoaderError::StreamWorkerPanic {
2447 worker,
2448 batch,
2449 sequence,
2450 logical_id,
2451 },
2452 }
2453 }
2454}
2455
2456impl<S, C, F, I, M, N> Iterator for StreamLoaderIter<'_, S, C, F, I, M, N>
2457where
2458 S: WorkerSourceFactory,
2459 F: TransformFactory<S::Sample> + Send + Sync + 'static,
2460 F::Transform: Send + 'static,
2461 TransformOutput<S, F>: Send + 'static,
2462 TransformFailure<S, F>: Send + 'static,
2463 F::Error: Send + 'static,
2464 C: Collate<TransformOutput<S, F>>,
2465 I: WorkerInit + Send + Sync + 'static,
2466 I::Error: Send + 'static,
2467 M: StreamMemoryPolicy + Send + 'static,
2468 N: StreamPinPolicy<S, C, F>,
2469{
2470 type Item = std::result::Result<C::Batch, StreamLoaderError<S, C, F, I>>;
2471
2472 fn next(&mut self) -> Option<Self::Item> {
2473 if self.exhausted {
2474 return None;
2475 }
2476 if let Some(error) = self.pending_error.take() {
2477 self.close(false);
2478 return Some(Err(error));
2479 }
2480 if self.source_complete {
2481 self.close(false);
2482 return None;
2483 }
2484 let mut deadline_armed = false;
2485 loop {
2486 if self.ordered
2487 && let Some(index) = self
2488 .completed
2489 .iter()
2490 .position(|(sequence, _)| *sequence == self.next_sequence)
2491 {
2492 let (_, buffered) = self.completed.swap_remove(index);
2493 match self.consume(buffered) {
2494 Ok(true) => return Some(self.finish_batch()),
2495 Ok(false) => continue,
2496 Err(error) => return Some(Err(error)),
2497 }
2498 }
2499
2500 if self.ordered && self.completed.len() == self.reassembly_capacity {
2501 let expected = self.next_sequence;
2502 let found = self
2503 .completed
2504 .iter()
2505 .map(|(sequence, _)| *sequence)
2506 .min()
2507 .expect("a full ordered reassembly window is nonempty");
2508 return Some(Err(self.protocol_error(
2509 Some(expected),
2510 format!(
2511 "missing sequence {expected}; bounded reassembly window is full before buffered sequence {found}"
2512 ),
2513 )));
2514 }
2515
2516 if self.ordered
2517 && self.run_context.byte_budget.is_some()
2518 && let Some(pool) = self.pool.as_ref()
2519 && let Some(front_waiters) = M::waiters(&self.front_waiters)
2520 && front_waiters
2521 .iter()
2522 .enumerate()
2523 .any(|(worker, _)| !pool.pool().terminal.get(worker).copied().unwrap_or(false))
2524 && front_waiters.iter().enumerate().all(|(worker, sequence)| {
2525 pool.pool().terminal.get(worker).copied().unwrap_or(false)
2526 || sequence.is_some_and(|sequence| sequence > self.next_sequence)
2527 })
2528 {
2529 let expected = self.next_sequence;
2530 let found = M::waiters(&self.front_waiters)
2531 .expect("enabled byte accounting has waiter state")
2532 .iter()
2533 .flatten()
2534 .copied()
2535 .min()
2536 .expect("a nonterminal byte-budget worker has a front waiter");
2537 return Some(Err(self.protocol_error(
2538 Some(expected),
2539 format!(
2540 "missing sequence {expected}; all active byte-budget workers are waiting at sequence {found} or later"
2541 ),
2542 )));
2543 }
2544
2545 if self
2546 .pool
2547 .as_ref()
2548 .is_some_and(|pool| pool.pool().all_terminal())
2549 {
2550 if self.ordered
2551 && let Some(found) = self.completed.iter().map(|(sequence, _)| *sequence).min()
2552 {
2553 let expected = self.next_sequence;
2554 return Some(Err(self.protocol_error(
2555 Some(expected),
2556 format!("missing sequence {expected} before buffered sequence {found}"),
2557 )));
2558 }
2559 if self.ordered
2560 && let Some(expected) = self.expected_records
2561 && self.next_sequence != expected as u64
2562 {
2563 let missing = self.next_sequence.min(expected as u64);
2564 return Some(Err(self.protocol_error(
2565 Some(missing),
2566 format!(
2567 "stream ended after {} ordered records, expected {expected}",
2568 self.next_sequence
2569 ),
2570 )));
2571 }
2572 self.source_complete = true;
2573 if self.partial.is_empty() || self.drop_last {
2574 self.close(false);
2575 return None;
2576 }
2577 return Some(self.finish_batch());
2578 }
2579
2580 if !deadline_armed && let Some(timeout) = self.timeout {
2581 self.run_context.deadline.arm(timeout);
2582 deadline_armed = true;
2583 }
2584 let remaining = self
2585 .timeout
2586 .and_then(|_| self.run_context.deadline.remaining());
2587 let received = self
2588 .pool
2589 .as_ref()
2590 .expect("active stream iterator has a pool")
2591 .pool()
2592 .receive(&self.run_context, remaining);
2593 let completion = match received {
2594 StreamReceive::Completion(completion) => completion,
2595 StreamReceive::Timeout => {
2596 let batch = self.next_batch;
2597 self.run_context.cancel();
2598 self.disarm_deadline();
2599 self.close(false);
2600 return Some(Err(LoaderError::Timeout { batch }));
2601 }
2602 StreamReceive::Cancelled => {
2603 self.disarm_deadline();
2604 self.close(false);
2605 return Some(Err(LoaderError::Cancelled));
2606 }
2607 StreamReceive::Closed => {
2608 let batch = self.next_batch;
2609 self.disarm_deadline();
2610 self.close(true);
2611 return Some(Err(LoaderError::ChannelClosed { batch }));
2612 }
2613 };
2614 if completion.generation != self.generation {
2615 if completion.holds_credit && self.return_credit(completion.worker).is_err() {
2616 self.close(true);
2617 return Some(Err(LoaderError::ChannelClosed {
2618 batch: self.next_batch,
2619 }));
2620 }
2621 continue;
2622 }
2623
2624 match completion.result {
2625 Err(failure) => {
2626 self.set_front_waiter(completion.worker, None);
2627 self.run_context.cancel();
2628 let fatal = matches!(
2629 &failure,
2630 StreamFailure::TransformInit(_)
2631 | StreamFailure::WorkerInit(_)
2632 | StreamFailure::Panic
2633 );
2634 if self
2635 .pool
2636 .as_mut()
2637 .expect("stream pool")
2638 .pool_mut()
2639 .account_parts(completion.worker, completion.holds_credit, fatal, fatal)
2640 .is_err()
2641 {
2642 self.close(true);
2643 return Some(Err(LoaderError::ChannelClosed {
2644 batch: self.next_batch,
2645 }));
2646 }
2647 let error = self.map_failure(
2648 completion.worker,
2649 completion.sequence,
2650 completion.logical_id,
2651 failure,
2652 );
2653 self.disarm_deadline();
2654 self.close(fatal);
2655 return Some(Err(error));
2656 }
2657 Ok(StreamMessage::End) => {
2658 self.set_front_waiter(completion.worker, None);
2659 if self
2660 .pool
2661 .as_mut()
2662 .expect("stream pool")
2663 .pool_mut()
2664 .account_parts(completion.worker, completion.holds_credit, true, false)
2665 .is_err()
2666 {
2667 self.close(true);
2668 return Some(Err(LoaderError::ChannelClosed {
2669 batch: self.next_batch,
2670 }));
2671 }
2672 }
2673 Ok(StreamMessage::Waiting(waiting)) => {
2674 self.set_front_waiter(completion.worker, Some(M::waiting_sequence(waiting)));
2675 }
2676 Ok(StreamMessage::Record(record, permit)) => {
2677 self.set_front_waiter(completion.worker, None);
2678 let buffered = BufferedRecord {
2679 worker: completion.worker,
2680 record,
2681 permit,
2682 };
2683 if !self.ordered {
2684 match self.consume(buffered) {
2685 Ok(true) => return Some(self.finish_batch()),
2686 Ok(false) => {}
2687 Err(error) => return Some(Err(error)),
2688 }
2689 continue;
2690 }
2691 let Some(sequence) = buffered.record.sequence else {
2692 self.run_context.cancel();
2693 if self.return_credit(buffered.worker).is_err() {
2694 self.close(true);
2695 return Some(Err(LoaderError::ChannelClosed {
2696 batch: self.next_batch,
2697 }));
2698 }
2699 return Some(Err(self.protocol_error(
2700 None,
2701 "ordered stream record did not provide a sequence identifier",
2702 )));
2703 };
2704 let sequence = sequence.get();
2705 if sequence < self.next_sequence {
2706 self.run_context.cancel();
2707 if self.return_credit(buffered.worker).is_err() {
2708 self.close(true);
2709 return Some(Err(LoaderError::ChannelClosed {
2710 batch: self.next_batch,
2711 }));
2712 }
2713 return Some(Err(self.protocol_error(
2714 Some(sequence),
2715 format!(
2716 "duplicate or past sequence {sequence}; next expected is {}",
2717 self.next_sequence
2718 ),
2719 )));
2720 }
2721 if self
2722 .completed
2723 .iter()
2724 .any(|(buffered_sequence, _)| *buffered_sequence == sequence)
2725 {
2726 self.run_context.cancel();
2727 if self.return_credit(buffered.worker).is_err() {
2728 self.close(true);
2729 return Some(Err(LoaderError::ChannelClosed {
2730 batch: self.next_batch,
2731 }));
2732 }
2733 return Some(Err(self.protocol_error(
2734 Some(sequence),
2735 format!("duplicate buffered sequence {sequence}"),
2736 )));
2737 }
2738 self.completed.push((sequence, buffered));
2739 }
2740 }
2741 }
2742 }
2743}
2744
2745impl<S, C, F, I, M, N> Drop for StreamLoaderIter<'_, S, C, F, I, M, N>
2746where
2747 S: WorkerSourceFactory,
2748 F: TransformFactory<S::Sample>,
2749 C: Collate<TransformOutput<S, F>>,
2750 I: WorkerInit,
2751 M: StreamMemoryPolicy,
2752{
2753 fn drop(&mut self) {
2754 self.run_context.cancel();
2755 let mut failed = false;
2756 while let Some((_, buffered)) = self.completed.pop() {
2757 failed |= self
2758 .pool
2759 .as_ref()
2760 .is_none_or(|pool| pool.pool().return_credit(buffered.worker).is_err());
2761 }
2762 if let Some(mut pool) = self.pool.take()
2763 && pool.is_persistent()
2764 && (failed || pool.pool_mut().quiesce(self.generation).is_err())
2765 {
2766 pool.pool_mut().poison();
2767 }
2768 }
2769}
2770
2771fn invalid_configuration(field: &'static str, reason: impl Into<String>) -> RustTorchError {
2772 RustTorchError::InvalidConfiguration {
2773 field,
2774 reason: reason.into(),
2775 }
2776}
2777
2778#[cfg(test)]
2779mod tests {
2780 use std::convert::Infallible;
2781
2782 use super::*;
2783
2784 type TestCompletion = StreamCompletion<
2785 usize,
2786 (),
2787 Infallible,
2788 Infallible,
2789 Infallible,
2790 Infallible,
2791 Infallible,
2792 Infallible,
2793 >;
2794 type TestResult = StreamResult<
2795 usize,
2796 (),
2797 Infallible,
2798 Infallible,
2799 Infallible,
2800 Infallible,
2801 Infallible,
2802 Infallible,
2803 >;
2804
2805 fn ready_completion(result: TestResult) -> TestCompletion {
2806 StreamCompletion {
2807 worker: 0,
2808 generation: 0,
2809 sequence: None,
2810 logical_id: None,
2811 holds_credit: false,
2812 result,
2813 }
2814 }
2815
2816 fn receive_after_expiry(completion: TestCompletion) -> TestCompletion {
2817 let (sender, receiver) = bounded(1);
2818 sender.send(completion).unwrap();
2819 let context = WorkerRunContext::new(0, 0, 0);
2820 context.deadline.arm(Duration::ZERO);
2821 let shutdown = CancellationToken::new();
2822 match receive_ready_first(&receiver, &context, &shutdown, Some(Duration::ZERO)) {
2823 ReadyFirst::Completion(completion) => completion,
2824 ReadyFirst::Timeout => panic!("ready completion lost to expired deadline"),
2825 ReadyFirst::Cancelled => panic!("ready completion lost to cancellation"),
2826 ReadyFirst::Closed => panic!("ready completion lost to channel close"),
2827 }
2828 }
2829
2830 #[test]
2831 fn queued_error_and_end_beat_an_expired_receive_deadline() {
2832 let error = receive_after_expiry(ready_completion(Err(StreamFailure::Panic)));
2833 assert!(matches!(error.result, Err(StreamFailure::Panic)));
2834
2835 let end = receive_after_expiry(ready_completion(Ok(StreamMessage::End)));
2836 assert!(matches!(end.result, Ok(StreamMessage::End)));
2837 }
2838}