1use std::{
15 convert::Infallible,
16 marker::PhantomData,
17 mem::{MaybeUninit, size_of},
18 num::NonZeroUsize,
19 panic::{AssertUnwindSafe, catch_unwind},
20 sync::{Arc, Mutex, atomic::AtomicUsize},
21 thread::{self, JoinHandle},
22 time::Duration,
23};
24
25use crossbeam_channel::{Receiver, RecvError, Sender, TryRecvError, after, bounded, select};
26
27use rusttorch_core::{Device, Result, RustTorchError, available_devices};
28
29use crate::memory::{BudgetError, ByteBudget, MemoryDisabled, MemoryEnabled, MemoryPolicy};
30use crate::worker::WorkerRunContext;
31#[path = "stream_checkpoint.rs"]
32mod exact;
33use crate::{
34 Auto, CancellationToken, CloneTransformFactory, Collate, Deadline, DefaultCollator, Explicit,
35 IdentityTransformFactory, LoaderError, MemoryFootprint, NoWorkerInit, PinDisabled, PinEnabled,
36 PinMemory, PinMemoryStatus, PipelineError, TaskContext, Transform, TransformFactory,
37 WorkerContext, WorkerInfo, WorkerInit, with_worker_info,
38};
39pub use exact::{ExactStreamDataLoader, ExactStreamLoaderIter};
40
41#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
48pub struct SequenceId(u64);
49
50impl SequenceId {
51 pub const fn new(value: u64) -> Self {
53 Self(value)
54 }
55
56 pub const fn get(self) -> u64 {
58 self.0
59 }
60
61 pub const fn checked_next(self) -> Option<Self> {
63 match self.0.checked_add(1) {
64 Some(value) => Some(Self(value)),
65 None => None,
66 }
67 }
68}
69
70#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
76pub struct LogicalSampleId(u64);
77
78impl LogicalSampleId {
79 pub const fn new(value: u64) -> Self {
81 Self(value)
82 }
83
84 pub const fn get(self) -> u64 {
86 self.0
87 }
88
89 pub const fn checked_next(self) -> Option<Self> {
91 match self.0.checked_add(1) {
92 Some(value) => Some(Self(value)),
93 None => None,
94 }
95 }
96}
97
98#[derive(Clone, Debug, Eq, PartialEq)]
104pub struct WorkerRecord<T> {
105 pub sequence: Option<SequenceId>,
107 pub logical_id: LogicalSampleId,
109 pub sample: T,
111}
112
113pub trait WorkerSourceFactory: Send + Sync + 'static {
126 type Sample: Send + 'static;
128 type Error: Send + 'static;
130 type Source: Iterator<Item = std::result::Result<WorkerRecord<Self::Sample>, Self::Error>>
132 + Send
133 + 'static;
134
135 fn create(&self, worker: WorkerContext) -> std::result::Result<Self::Source, Self::Error>;
137
138 fn exact_len(&self) -> Option<usize> {
140 None
141 }
142}
143
144type TransformOutput<S, F> =
145 <<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<
146 <S as WorkerSourceFactory>::Sample,
147 >>::Output;
148type TransformFailure<S, F> =
149 <<F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Transform as Transform<
150 <S as WorkerSourceFactory>::Sample,
151 >>::Error;
152type StreamPipelineError<S, C, F, I> = PipelineError<
153 <S as WorkerSourceFactory>::Error,
154 TransformFailure<S, F>,
155 <C as Collate<TransformOutput<S, F>>>::Error,
156 <F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Error,
157 <I as WorkerInit>::Error,
158>;
159type StreamLoaderError<S, C, F, I> = LoaderError<StreamPipelineError<S, C, F, I>>;
160type ReassemblyEntry<T, P> = (u64, BufferedRecord<T, P>);
161type RetainedReassembly<T, P> = Mutex<Vec<ReassemblyEntry<T, P>>>;
162type StreamFootprint<S, F> = fn(&TransformOutput<S, F>) -> usize;
163
164enum StreamByteFailure {
165 MemoryLimit {
166 limit: usize,
167 actual: usize,
168 },
169 Protocol {
170 sequence: Option<u64>,
171 reason: String,
172 },
173}
174
175trait StreamMemoryPolicy: MemoryPolicy {
176 type Waiters;
177 type Waiting: Send + 'static;
178 type Failure: Send + 'static;
179
180 fn waiter_bytes(workers: usize) -> Result<usize>;
181 fn preflight_waiters(workers: usize) -> Result<()>;
182 fn new_waiters(workers: usize) -> Result<Self::Waiters>;
183 fn waiters(waiters: &Self::Waiters) -> Option<&[Option<u64>]>;
184 fn waiters_mut(waiters: &mut Self::Waiters) -> Option<&mut [Option<u64>]>;
185 fn waiting(sequence: u64) -> Option<Self::Waiting>;
186 fn waiting_sequence(waiting: Self::Waiting) -> u64;
187 fn memory_limit(limit: usize, actual: usize) -> Option<Self::Failure>;
188 fn protocol(sequence: Option<u64>, reason: String) -> Option<Self::Failure>;
189 fn map_failure<E>(
190 failure: Self::Failure,
191 worker: usize,
192 sequence: Option<u64>,
193 logical_id: Option<u64>,
194 ) -> LoaderError<E>;
195}
196
197impl StreamMemoryPolicy for MemoryDisabled {
198 type Waiters = ();
199 type Waiting = Infallible;
200 type Failure = Infallible;
201
202 fn waiter_bytes(_workers: usize) -> Result<usize> {
203 Ok(0)
204 }
205
206 fn preflight_waiters(_workers: usize) -> Result<()> {
207 Ok(())
208 }
209
210 fn new_waiters(_workers: usize) -> Result<Self::Waiters> {
211 Ok(())
212 }
213
214 fn waiters(_waiters: &Self::Waiters) -> Option<&[Option<u64>]> {
215 None
216 }
217
218 fn waiters_mut(_waiters: &mut Self::Waiters) -> Option<&mut [Option<u64>]> {
219 None
220 }
221
222 fn waiting(_sequence: u64) -> Option<Self::Waiting> {
223 None
224 }
225
226 fn waiting_sequence(waiting: Self::Waiting) -> u64 {
227 match waiting {}
228 }
229
230 fn memory_limit(_limit: usize, _actual: usize) -> Option<Self::Failure> {
231 None
232 }
233
234 fn protocol(_sequence: Option<u64>, _reason: String) -> Option<Self::Failure> {
235 None
236 }
237
238 fn map_failure<E>(
239 failure: Self::Failure,
240 _worker: usize,
241 _sequence: Option<u64>,
242 _logical_id: Option<u64>,
243 ) -> LoaderError<E> {
244 match failure {}
245 }
246}
247
248impl StreamMemoryPolicy for MemoryEnabled {
249 type Waiters = Vec<Option<u64>>;
250 type Waiting = u64;
251 type Failure = StreamByteFailure;
252
253 fn waiter_bytes(workers: usize) -> Result<usize> {
254 workers
255 .checked_mul(size_of::<Option<u64>>())
256 .ok_or_else(|| capacity_error("stream byte waiter storage exceeds usize"))
257 }
258
259 fn preflight_waiters(workers: usize) -> Result<()> {
260 preflight_vec::<Option<u64>>(workers, "stream byte waiter state")
261 }
262
263 fn new_waiters(workers: usize) -> Result<Self::Waiters> {
264 let mut waiters = Vec::new();
265 reserve_exact(&mut waiters, workers, "stream byte waiter state")?;
266 waiters.resize(workers, None);
267 Ok(waiters)
268 }
269
270 fn waiters(waiters: &Self::Waiters) -> Option<&[Option<u64>]> {
271 Some(waiters)
272 }
273
274 fn waiters_mut(waiters: &mut Self::Waiters) -> Option<&mut [Option<u64>]> {
275 Some(waiters)
276 }
277
278 fn waiting(sequence: u64) -> Option<Self::Waiting> {
279 Some(sequence)
280 }
281
282 fn waiting_sequence(waiting: Self::Waiting) -> u64 {
283 waiting
284 }
285
286 fn memory_limit(limit: usize, actual: usize) -> Option<Self::Failure> {
287 Some(StreamByteFailure::MemoryLimit { limit, actual })
288 }
289
290 fn protocol(sequence: Option<u64>, reason: String) -> Option<Self::Failure> {
291 Some(StreamByteFailure::Protocol { sequence, reason })
292 }
293
294 fn map_failure<E>(
295 failure: Self::Failure,
296 worker: usize,
297 sequence: Option<u64>,
298 logical_id: Option<u64>,
299 ) -> LoaderError<E> {
300 match failure {
301 StreamByteFailure::MemoryLimit { limit, actual } => LoaderError::MemoryLimit {
302 batch: None,
303 worker: Some(worker),
304 sequence,
305 logical_id,
306 limit,
307 actual,
308 },
309 StreamByteFailure::Protocol { sequence, reason } => {
310 LoaderError::StreamProtocol { sequence, reason }
311 }
312 }
313 }
314}
315
316#[doc(hidden)]
317pub trait StreamPinPolicy<S, C, F>
318where
319 S: WorkerSourceFactory,
320 F: TransformFactory<S::Sample>,
321 C: Collate<TransformOutput<S, F>>,
322{
323 fn pin(batch: C::Batch, device: Device) -> Result<C::Batch>;
324}
325
326impl<S, C, F> StreamPinPolicy<S, C, F> for PinDisabled
327where
328 S: WorkerSourceFactory,
329 F: TransformFactory<S::Sample>,
330 C: Collate<TransformOutput<S, F>>,
331{
332 fn pin(batch: C::Batch, _device: Device) -> Result<C::Batch> {
333 Ok(batch)
334 }
335}
336
337impl<S, C, F, Q> StreamPinPolicy<S, C, F> for PinEnabled<Q>
338where
339 S: WorkerSourceFactory,
340 F: TransformFactory<S::Sample>,
341 C: Collate<TransformOutput<S, F>>,
342 C::Batch: PinMemory,
343{
344 fn pin(batch: C::Batch, device: Device) -> Result<C::Batch> {
345 batch.pin_memory(device)
346 }
347}
348
349type StreamCompletionFor<S, F, I, M> = StreamCompletion<
350 TransformOutput<S, F>,
351 <M as MemoryPolicy>::Permit,
352 <M as StreamMemoryPolicy>::Waiting,
353 <M as StreamMemoryPolicy>::Failure,
354 <S as WorkerSourceFactory>::Error,
355 TransformFailure<S, F>,
356 <F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Error,
357 <I as WorkerInit>::Error,
358>;
359type StreamFailureFor<S, F, I, M> = StreamFailure<
360 <S as WorkerSourceFactory>::Error,
361 TransformFailure<S, F>,
362 <F as TransformFactory<<S as WorkerSourceFactory>::Sample>>::Error,
363 <I as WorkerInit>::Error,
364 <M as StreamMemoryPolicy>::Failure,
365>;
366type StreamResult<T, P, W, ME, SE, TE, FE, IE> =
367 std::result::Result<StreamMessage<T, P, W>, StreamFailure<SE, TE, FE, IE, ME>>;
368type StreamCompletionSender<T, P, W, ME, SE, TE, FE, IE> =
369 Sender<StreamCompletion<T, P, W, ME, SE, TE, FE, IE>>;
370
371enum StreamFailure<SE, TE, FE, IE, ME> {
372 Source(SE),
373 Transform(TE),
374 TransformInit(FE),
375 WorkerInit(IE),
376 Memory(ME),
377 Panic,
378}
379
380enum StreamMessage<T, P, W> {
381 Record(WorkerRecord<T>, P),
382 Waiting(W),
383 End,
384}
385
386struct StreamCompletion<T, P, W, ME, SE, TE, FE, IE> {
387 worker: usize,
388 generation: u64,
389 sequence: Option<u64>,
390 logical_id: Option<u64>,
391 holds_credit: bool,
392 result: StreamResult<T, P, W, ME, SE, TE, FE, IE>,
393}
394
395enum StreamControl {
396 Begin(WorkerRunContext),
397}
398
399enum StreamReceive<S, F, I, M>
400where
401 S: WorkerSourceFactory,
402 F: TransformFactory<S::Sample>,
403 I: WorkerInit,
404 M: StreamMemoryPolicy,
405{
406 Completion(StreamCompletionFor<S, F, I, M>),
407 Timeout,
408 Cancelled,
409 Closed,
410}
411
412enum ReadyFirst<T> {
413 Completion(T),
414 Timeout,
415 Cancelled,
416 Closed,
417}
418
419const MAX_STREAM_QUEUE_ALLOCATION_BYTES: usize = 64 * 1024 * 1024;
420const CHANNEL_CONTROL_BLOCK_ALLOWANCE_BYTES: usize = 2 * 1024;
421
422#[allow(dead_code)]
423#[repr(C)]
424struct CapacitySlot<T> {
425 stamp: AtomicUsize,
426 message: MaybeUninit<T>,
427}
428
429fn validate_stream_capacity<S, F, I, M>(
430 workers: usize,
431 prefetch_factor: usize,
432 outstanding: usize,
433 batch_size: usize,
434 reassembly_slots: usize,
435) -> Result<()>
436where
437 S: WorkerSourceFactory,
438 F: TransformFactory<S::Sample>,
439 I: WorkerInit,
440 M: StreamMemoryPolicy,
441{
442 validate_stream_capacity_extra::<S, F, I, M>(
443 workers,
444 prefetch_factor,
445 outstanding,
446 batch_size,
447 reassembly_slots,
448 0,
449 )
450}
451
452fn validate_stream_capacity_extra<S, F, I, M>(
453 workers: usize,
454 prefetch_factor: usize,
455 outstanding: usize,
456 batch_size: usize,
457 reassembly_slots: usize,
458 extra: usize,
459) -> Result<()>
460where
461 S: WorkerSourceFactory,
462 F: TransformFactory<S::Sample>,
463 I: WorkerInit,
464 M: StreamMemoryPolicy,
465{
466 let expected = workers
467 .checked_mul(prefetch_factor)
468 .ok_or_else(|| capacity_error("workers multiplied by prefetch_factor exceeds usize"))?;
469 if expected != outstanding {
470 return Err(capacity_error(
471 "outstanding capacity does not match workers multiplied by prefetch_factor",
472 ));
473 }
474 for capacity in [1, prefetch_factor, outstanding] {
475 capacity
476 .checked_add(1)
477 .and_then(usize::checked_next_power_of_two)
478 .and_then(|mark_bit| mark_bit.checked_mul(2))
479 .ok_or_else(|| capacity_error("bounded channel ring arithmetic overflowed"))?;
480 }
481
482 let credit_bytes = outstanding
483 .checked_mul(size_of::<CapacitySlot<()>>())
484 .ok_or_else(|| capacity_error("stream credit storage exceeds usize"))?;
485 let result_bytes = outstanding
486 .checked_mul(size_of::<CapacitySlot<StreamCompletionFor<S, F, I, M>>>())
487 .ok_or_else(|| capacity_error("stream result storage exceeds usize"))?;
488 let control_bytes = workers
489 .checked_mul(size_of::<CapacitySlot<StreamControl>>())
490 .ok_or_else(|| capacity_error("stream control storage exceeds usize"))?;
491 let batch_bytes = batch_size
492 .checked_mul(size_of::<TransformOutput<S, F>>())
493 .and_then(|bytes| bytes.checked_mul(2))
494 .ok_or_else(|| capacity_error("stream batch storage exceeds usize"))?;
495 let reassembly_bytes = reassembly_slots
496 .checked_mul(size_of::<
497 ReassemblyEntry<TransformOutput<S, F>, <M as MemoryPolicy>::Permit>,
498 >())
499 .ok_or_else(|| capacity_error("ordered stream reassembly exceeds usize"))?;
500 let per_worker = size_of::<WorkerInfo>()
501 .checked_add(size_of::<(Sender<()>, Receiver<()>)>())
502 .and_then(|bytes| {
503 bytes.checked_add(size_of::<(Sender<StreamControl>, Receiver<StreamControl>)>())
504 })
505 .and_then(|bytes| bytes.checked_add(size_of::<Sender<()>>()))
506 .and_then(|bytes| bytes.checked_add(size_of::<Sender<StreamControl>>()))
507 .and_then(|bytes| bytes.checked_add(size_of::<JoinHandle<()>>()))
508 .and_then(|bytes| bytes.checked_add(size_of::<bool>()))
509 .ok_or_else(|| capacity_error("stream worker bookkeeping exceeds usize"))?;
510 let waiter_bytes = M::waiter_bytes(workers)?;
511 let bookkeeping = workers
512 .checked_mul(per_worker)
513 .and_then(|bytes| bytes.checked_add(waiter_bytes))
514 .ok_or_else(|| capacity_error("stream worker bookkeeping exceeds usize"))?;
515 let channel_count = workers
516 .checked_mul(2)
517 .and_then(|count| count.checked_add(1))
518 .ok_or_else(|| capacity_error("stream channel count exceeds usize"))?;
519 let channel_bytes = channel_count
520 .checked_mul(CHANNEL_CONTROL_BLOCK_ALLOWANCE_BYTES)
521 .ok_or_else(|| capacity_error("stream channel controls exceed usize"))?;
522 let aggregate = credit_bytes
523 .checked_add(result_bytes)
524 .and_then(|bytes| bytes.checked_add(control_bytes))
525 .and_then(|bytes| bytes.checked_add(batch_bytes))
526 .and_then(|bytes| bytes.checked_add(reassembly_bytes))
527 .and_then(|bytes| bytes.checked_add(bookkeeping))
528 .and_then(|bytes| bytes.checked_add(channel_bytes))
529 .and_then(|bytes| bytes.checked_add(extra))
530 .ok_or_else(|| capacity_error("aggregate stream queue storage exceeds usize"))?;
531 if aggregate > MAX_STREAM_QUEUE_ALLOCATION_BYTES {
532 return Err(capacity_error(format!(
533 "aggregate stream queue storage requires {aggregate} bytes, above the {MAX_STREAM_QUEUE_ALLOCATION_BYTES}-byte safety ceiling"
534 )));
535 }
536 preflight_channel_storage::<()>(outstanding)?;
537 preflight_channel_storage::<StreamControl>(workers)?;
538 preflight_channel_storage::<StreamCompletionFor<S, F, I, M>>(outstanding)?;
539 preflight_vec::<TransformOutput<S, F>>(batch_size, "stream batch")?;
540 M::preflight_waiters(workers)?;
541 Ok(())
542}
543
544fn preflight_channel_storage<T>(capacity: usize) -> Result<()> {
545 let mut reservation: Vec<MaybeUninit<CapacitySlot<T>>> = Vec::new();
546 reservation
547 .try_reserve_exact(capacity)
548 .map_err(|error| capacity_error(format!("bounded stream storage is unavailable: {error}")))
549}
550
551fn preflight_vec<T>(capacity: usize, label: &'static str) -> Result<()> {
552 let mut reservation: Vec<T> = Vec::new();
553 reservation
554 .try_reserve_exact(capacity)
555 .map_err(|error| capacity_error(format!("{label} capacity is unavailable: {error}")))
556}
557
558fn capacity_error(reason: impl Into<String>) -> RustTorchError {
559 invalid_configuration("prefetch_factor", reason)
560}
561
562struct StreamWorkerPool<S, F, I, M>
563where
564 S: WorkerSourceFactory,
565 F: TransformFactory<S::Sample>,
566 I: WorkerInit,
567 M: StreamMemoryPolicy,
568{
569 controls: Vec<Sender<StreamControl>>,
570 credits: Vec<Sender<()>>,
571 results: Option<Receiver<StreamCompletionFor<S, F, I, M>>>,
572 handles: Vec<JoinHandle<()>>,
573 shutdown: CancellationToken,
574 active: Option<WorkerRunContext>,
575 terminal: Vec<bool>,
576 terminal_count: usize,
577 fatal_exit: bool,
578 poisoned: bool,
579}
580
581impl<S, F, I, M> StreamWorkerPool<S, F, I, M>
582where
583 S: WorkerSourceFactory,
584 F: TransformFactory<S::Sample> + Send + Sync + 'static,
585 F::Transform: Send + 'static,
586 TransformOutput<S, F>: Send + 'static,
587 TransformFailure<S, F>: Send + 'static,
588 F::Error: Send + 'static,
589 I: WorkerInit + Send + Sync + 'static,
590 I::Error: Send + 'static,
591 M: StreamMemoryPolicy + Send + 'static,
592{
593 #[allow(clippy::too_many_arguments)]
594 fn new(
595 source_factory: Arc<S>,
596 transform_factory: Arc<F>,
597 initializer: Arc<I>,
598 configuration: StreamConfiguration,
599 generation: u64,
600 outstanding: usize,
601 reassembly_slots: usize,
602 footprint: Option<StreamFootprint<S, F>>,
603 ) -> Result<Self> {
604 validate_stream_capacity::<S, F, I, M>(
605 configuration.workers,
606 configuration.prefetch_factor,
607 outstanding,
608 configuration.batch_size,
609 reassembly_slots,
610 )?;
611
612 let (result_sender, result_receiver) =
613 bounded_checked(outstanding, "stream result channel")?;
614 let mut credit_lanes = Vec::new();
615 let mut control_lanes = Vec::new();
616 reserve_exact(
617 &mut credit_lanes,
618 configuration.workers,
619 "stream credit channels",
620 )?;
621 reserve_exact(
622 &mut control_lanes,
623 configuration.workers,
624 "stream control channels",
625 )?;
626 for _ in 0..configuration.workers {
627 let (credit_sender, credit_receiver) =
628 bounded_checked(configuration.prefetch_factor, "stream credit channel")?;
629 for _ in 0..configuration.prefetch_factor {
630 credit_sender.try_send(()).map_err(|_| {
631 capacity_error("validated stream credit channel rejected a token")
632 })?;
633 }
634 credit_lanes.push((credit_sender, credit_receiver));
635 control_lanes.push(bounded_checked(1, "stream control channel")?);
636 }
637
638 let shutdown = CancellationToken::new();
639 let mut pool = Self {
640 controls: Vec::new(),
641 credits: Vec::new(),
642 results: Some(result_receiver),
643 handles: Vec::new(),
644 shutdown: shutdown.clone(),
645 active: None,
646 terminal: Vec::new(),
647 terminal_count: 0,
648 fatal_exit: false,
649 poisoned: false,
650 };
651 reserve_exact(&mut pool.controls, configuration.workers, "stream controls")?;
652 reserve_exact(&mut pool.credits, configuration.workers, "stream credits")?;
653 reserve_exact(&mut pool.handles, configuration.workers, "stream workers")?;
654 reserve_exact(
655 &mut pool.terminal,
656 configuration.workers,
657 "stream terminal state",
658 )?;
659 pool.terminal.resize(configuration.workers, false);
660
661 for (id, ((credit_sender, credit_receiver), (control_sender, control_receiver))) in
662 credit_lanes.into_iter().zip(control_lanes).enumerate()
663 {
664 let info = WorkerInfo::from_loader_seed(
665 id,
666 configuration.workers,
667 configuration.loader_seed,
668 configuration.rank,
669 generation,
670 )?;
671 pool.credits.push(credit_sender);
672 pool.controls.push(control_sender);
673 let source_factory = Arc::clone(&source_factory);
674 let transform_factory = Arc::clone(&transform_factory);
675 let initializer = Arc::clone(&initializer);
676 let results = result_sender.clone();
677 let shutdown = shutdown.clone();
678 let handle = thread::Builder::new()
679 .name(format!("rusttorch-stream-worker-{id}"))
680 .spawn(move || {
681 run_stream_worker::<S, F, I, M>(
682 info,
683 generation,
684 source_factory,
685 transform_factory,
686 initializer,
687 control_receiver,
688 credit_receiver,
689 results,
690 shutdown,
691 footprint,
692 configuration.ordered,
693 );
694 })
695 .map_err(|error| RustTorchError::BackendUnavailable {
696 backend: "data loader stream worker threads",
697 reason: error.to_string(),
698 })?;
699 pool.handles.push(handle);
700 }
701 drop(result_sender);
702 Ok(pool)
703 }
704}
705
706impl<S, F, I, M> StreamWorkerPool<S, F, I, M>
707where
708 S: WorkerSourceFactory,
709 F: TransformFactory<S::Sample>,
710 I: WorkerInit,
711 M: StreamMemoryPolicy,
712{
713 fn start_generation(&mut self, context: WorkerRunContext) -> std::result::Result<(), ()> {
714 if self.poisoned || self.active.is_some() || self.shutdown.is_cancelled() {
715 return Err(());
716 }
717 self.terminal.fill(false);
718 self.terminal_count = 0;
719 self.fatal_exit = false;
720 self.active = Some(context.clone());
721 for control in &self.controls {
722 if control.send(StreamControl::Begin(context.clone())).is_err() {
723 self.poisoned = true;
724 self.shutdown();
725 return Err(());
726 }
727 }
728 Ok(())
729 }
730
731 fn receive(
732 &self,
733 context: &WorkerRunContext,
734 timeout: Option<Duration>,
735 ) -> StreamReceive<S, F, I, M> {
736 let results = self
737 .results
738 .as_ref()
739 .expect("stream result receiver exists before shutdown");
740 match receive_ready_first(results, context, &self.shutdown, timeout) {
741 ReadyFirst::Completion(completion) => StreamReceive::Completion(completion),
742 ReadyFirst::Timeout => StreamReceive::Timeout,
743 ReadyFirst::Cancelled => StreamReceive::Cancelled,
744 ReadyFirst::Closed => StreamReceive::Closed,
745 }
746 }
747
748 fn account(
749 &mut self,
750 completion: &StreamCompletionFor<S, F, I, M>,
751 ) -> std::result::Result<(), ()> {
752 let fatal = matches!(
753 &completion.result,
754 Err(StreamFailure::TransformInit(_)
755 | StreamFailure::WorkerInit(_)
756 | StreamFailure::Panic)
757 );
758 let terminal = fatal || matches!(&completion.result, Ok(StreamMessage::End));
759 self.account_parts(completion.worker, completion.holds_credit, terminal, fatal)
760 }
761
762 fn account_parts(
763 &mut self,
764 worker: usize,
765 holds_credit: bool,
766 terminal: bool,
767 fatal: bool,
768 ) -> std::result::Result<(), ()> {
769 if holds_credit {
770 self.return_credit(worker)?;
771 }
772 if fatal {
773 self.fatal_exit = true;
774 }
775 if terminal {
776 let Some(seen) = self.terminal.get_mut(worker) else {
777 return Err(());
778 };
779 if !*seen {
780 *seen = true;
781 self.terminal_count += 1;
782 }
783 }
784 Ok(())
785 }
786
787 fn return_credit(&self, worker: usize) -> std::result::Result<(), ()> {
788 let Some(credit) = self.credits.get(worker) else {
789 return Err(());
790 };
791 credit.try_send(()).map_err(|_| ())
792 }
793
794 fn all_terminal(&self) -> bool {
795 self.terminal_count == self.terminal.len()
796 }
797
798 fn quiesce(&mut self, generation: u64) -> std::result::Result<(), ()> {
799 let Some(active) = self.active.take() else {
800 return Ok(());
801 };
802 active.cancel();
803 while !self.all_terminal() {
804 let completion = self.results.as_ref().ok_or(())?.recv().map_err(|_| ())?;
805 if completion.generation == generation {
806 self.account(&completion)?;
807 } else if completion.holds_credit {
808 self.return_credit(completion.worker)?;
809 }
810 }
811 while let Some(results) = self.results.as_ref() {
812 match results.try_recv() {
813 Ok(completion) => self.account(&completion)?,
814 Err(TryRecvError::Empty) => break,
815 Err(TryRecvError::Disconnected) => return Err(()),
816 }
817 }
818 if self.fatal_exit {
819 self.poisoned = true;
820 self.shutdown();
821 return Err(());
822 }
823 Ok(())
824 }
825
826 fn poison(&mut self) {
827 self.poisoned = true;
828 self.shutdown();
829 }
830
831 fn shutdown(&mut self) {
832 if let Some(active) = self.active.take() {
833 active.cancel();
834 }
835 self.shutdown.cancel();
836 self.controls.clear();
837 self.credits.clear();
838 self.results.take();
839 for handle in self.handles.drain(..) {
840 let _ = handle.join();
841 }
842 }
843}
844
845impl<S, F, I, M> Drop for StreamWorkerPool<S, F, I, M>
846where
847 S: WorkerSourceFactory,
848 F: TransformFactory<S::Sample>,
849 I: WorkerInit,
850 M: StreamMemoryPolicy,
851{
852 fn drop(&mut self) {
853 self.shutdown();
854 }
855}
856
857fn receive_ready_first<T>(
858 results: &Receiver<T>,
859 context: &WorkerRunContext,
860 shutdown: &CancellationToken,
861 timeout: Option<Duration>,
862) -> ReadyFirst<T> {
863 match results.try_recv() {
864 Ok(completion) => return ReadyFirst::Completion(completion),
865 Err(TryRecvError::Disconnected) => return ReadyFirst::Closed,
866 Err(TryRecvError::Empty) => {}
867 }
868 if let Some(timeout) = timeout {
869 let timer = after(timeout);
870 select! {
871 recv(results) -> result => map_ready_receive(result),
872 recv(context.cancellation.signal()) -> _ => match results.try_recv() {
873 Ok(completion) => ReadyFirst::Completion(completion),
874 Err(TryRecvError::Empty) => ReadyFirst::Cancelled,
875 Err(TryRecvError::Disconnected) => ReadyFirst::Closed,
876 },
877 recv(shutdown.signal()) -> _ => ReadyFirst::Closed,
878 recv(timer) -> _ => match results.try_recv() {
879 Ok(completion) => ReadyFirst::Completion(completion),
880 Err(TryRecvError::Empty) => ReadyFirst::Timeout,
881 Err(TryRecvError::Disconnected) => ReadyFirst::Closed,
882 },
883 }
884 } else {
885 select! {
886 recv(results) -> result => map_ready_receive(result),
887 recv(context.cancellation.signal()) -> _ => match results.try_recv() {
888 Ok(completion) => ReadyFirst::Completion(completion),
889 Err(TryRecvError::Empty) => ReadyFirst::Cancelled,
890 Err(TryRecvError::Disconnected) => ReadyFirst::Closed,
891 },
892 recv(shutdown.signal()) -> _ => ReadyFirst::Closed,
893 }
894 }
895}
896
897fn map_ready_receive<T>(result: std::result::Result<T, RecvError>) -> ReadyFirst<T> {
898 match result {
899 Ok(completion) => ReadyFirst::Completion(completion),
900 Err(_) => ReadyFirst::Closed,
901 }
902}
903
904fn reserve_exact<T>(values: &mut Vec<T>, capacity: usize, label: &'static str) -> Result<()> {
905 values
906 .try_reserve_exact(capacity)
907 .map_err(|error| capacity_error(format!("{label} capacity is unavailable: {error}")))
908}
909
910fn bounded_checked<T>(capacity: usize, label: &'static str) -> Result<(Sender<T>, Receiver<T>)> {
911 catch_unwind(AssertUnwindSafe(|| bounded(capacity))).map_err(|_| {
912 capacity_error(format!(
913 "{label} rejected the validated capacity {capacity}"
914 ))
915 })
916}
917
918#[allow(clippy::too_many_arguments)]
919fn run_stream_worker<S, F, I, M>(
920 info: WorkerInfo,
921 initialization_generation: u64,
922 source_factory: Arc<S>,
923 transform_factory: Arc<F>,
924 initializer: Arc<I>,
925 controls: Receiver<StreamControl>,
926 credits: Receiver<()>,
927 results: Sender<StreamCompletionFor<S, F, I, M>>,
928 shutdown: CancellationToken,
929 footprint: Option<StreamFootprint<S, F>>,
930 ordered: bool,
931) where
932 S: WorkerSourceFactory,
933 F: TransformFactory<S::Sample> + Send + Sync + 'static,
934 F::Transform: Send + 'static,
935 TransformOutput<S, F>: Send + 'static,
936 TransformFailure<S, F>: Send + 'static,
937 F::Error: Send + 'static,
938 I: WorkerInit + Send + Sync + 'static,
939 I::Error: Send + 'static,
940 M: StreamMemoryPolicy + Send + 'static,
941{
942 let lifecycle = WorkerContext::new(info, shutdown.clone(), shutdown.paired_deadline());
943 let mut active: Option<(WorkerRunContext, Option<u64>, Option<u64>, bool)> = None;
944 let outcome = catch_unwind(AssertUnwindSafe(|| {
945 with_worker_info(info, || {
946 if let Err(error) = initializer.initialize(&lifecycle) {
947 send_stream_failure(
948 &results,
949 &shutdown,
950 info.id,
951 initialization_generation,
952 None,
953 None,
954 false,
955 StreamFailure::WorkerInit(error),
956 );
957 return true;
958 }
959 let mut transform = match transform_factory.create(Some(&lifecycle)) {
960 Ok(transform) => transform,
961 Err(error) => {
962 send_stream_failure(
963 &results,
964 &shutdown,
965 info.id,
966 initialization_generation,
967 None,
968 None,
969 false,
970 StreamFailure::TransformInit(error),
971 );
972 return true;
973 }
974 };
975
976 loop {
977 let run = select! {
978 recv(controls) -> control => match control {
979 Ok(StreamControl::Begin(run)) => run,
980 Err(_) => return false,
981 },
982 recv(shutdown.signal()) -> _ => return false,
983 };
984 active = Some((run.clone(), None, None, false));
985 run_stream_generation::<S, F, I, M>(
986 info,
987 &source_factory,
988 &mut transform,
989 &credits,
990 &results,
991 &shutdown,
992 &run,
993 &mut active,
994 footprint,
995 ordered,
996 );
997 active = None;
998 if shutdown.is_cancelled() {
999 return false;
1000 }
1001 }
1002 })
1003 }));
1004 let fatal = match outcome {
1005 Ok(fatal) => fatal,
1006 Err(_) => {
1007 let (generation, sequence, logical_id, holds_credit) = active
1008 .as_ref()
1009 .map(|(run, sequence, logical_id, held)| {
1010 (run.generation, *sequence, *logical_id, *held)
1011 })
1012 .unwrap_or((initialization_generation, None, None, false));
1013 send_stream_failure(
1014 &results,
1015 &shutdown,
1016 info.id,
1017 generation,
1018 sequence,
1019 logical_id,
1020 holds_credit,
1021 StreamFailure::Panic,
1022 );
1023 true
1024 }
1025 };
1026 if fatal {
1027 shutdown.wait_cancelled();
1028 }
1029}
1030
1031#[allow(clippy::too_many_arguments)]
1032fn run_stream_generation<S, F, I, M>(
1033 info: WorkerInfo,
1034 source_factory: &S,
1035 transform: &mut F::Transform,
1036 credits: &Receiver<()>,
1037 results: &Sender<StreamCompletionFor<S, F, I, M>>,
1038 shutdown: &CancellationToken,
1039 run: &WorkerRunContext,
1040 active: &mut Option<(WorkerRunContext, Option<u64>, Option<u64>, bool)>,
1041 footprint: Option<StreamFootprint<S, F>>,
1042 ordered: bool,
1043) where
1044 S: WorkerSourceFactory,
1045 F: TransformFactory<S::Sample> + Send + Sync + 'static,
1046 F::Transform: Send + 'static,
1047 TransformOutput<S, F>: Send + 'static,
1048 TransformFailure<S, F>: Send + 'static,
1049 F::Error: Send + 'static,
1050 I: WorkerInit + Send + Sync + 'static,
1051 I::Error: Send + 'static,
1052 M: StreamMemoryPolicy,
1053{
1054 let generation_info = WorkerInfo::from_loader_seed(
1055 info.id,
1056 info.num_workers,
1057 run.loader_seed,
1058 info.rank,
1059 run.generation,
1060 )
1061 .expect("validated stream worker identity remains valid for every generation");
1062 let context = WorkerContext::new(
1063 generation_info,
1064 run.cancellation.clone(),
1065 run.deadline.clone(),
1066 );
1067 let mut source = match source_factory.create(context.clone()) {
1068 Ok(source) if context.check().is_ok() => source,
1069 Ok(source) => {
1070 drop(source);
1071 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, false);
1072 return;
1073 }
1074 Err(error) => {
1075 send_stream_failure(
1076 results,
1077 shutdown,
1078 info.id,
1079 run.generation,
1080 None,
1081 None,
1082 false,
1083 StreamFailure::Source(error),
1084 );
1085 run.cancel();
1086 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, false);
1087 return;
1088 }
1089 };
1090
1091 let mut last_sequence = None;
1092 loop {
1093 let acquired = select! {
1094 recv(credits) -> credit => credit.is_ok(),
1095 recv(run.cancellation.signal()) -> _ => false,
1096 recv(shutdown.signal()) -> _ => return,
1097 };
1098 if !acquired {
1099 drop(source);
1100 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, false);
1101 return;
1102 }
1103 active.as_mut().expect("active stream generation").3 = true;
1104 if context.check().is_err() {
1105 drop(source);
1106 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1107 active.as_mut().expect("active stream generation").3 = false;
1108 return;
1109 }
1110
1111 let record = match source.next() {
1112 Some(Ok(record)) => {
1113 if context.check().is_err() {
1114 drop(source);
1115 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1116 active.as_mut().expect("active stream generation").3 = false;
1117 return;
1118 }
1119 record
1120 }
1121 None => {
1122 drop(source);
1123 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1124 active.as_mut().expect("active stream generation").3 = false;
1125 return;
1126 }
1127 Some(Err(error)) => {
1128 drop(source);
1129 send_stream_failure(
1130 results,
1131 shutdown,
1132 info.id,
1133 run.generation,
1134 None,
1135 None,
1136 true,
1137 StreamFailure::Source(error),
1138 );
1139 active.as_mut().expect("active stream generation").3 = false;
1140 run.cancel();
1141 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, false);
1142 return;
1143 }
1144 };
1145 let sequence = record.sequence.map(SequenceId::get);
1146 let logical_id = record.logical_id.get();
1147 {
1148 let active = active.as_mut().expect("active stream generation");
1149 active.1 = sequence;
1150 active.2 = Some(logical_id);
1151 }
1152 let task_context = TaskContext {
1153 loader_seed: run.loader_seed,
1154 epoch: run.epoch,
1155 rank: info.rank,
1156 logical_sample: logical_id,
1157 stage: 0,
1158 cancellation: run.cancellation.clone(),
1159 deadline: run.deadline.clone(),
1160 };
1161 let sample = match transform.transform(record.sample, &task_context) {
1162 Ok(sample) if context.check().is_ok() => sample,
1163 Ok(_) => {
1164 drop(source);
1165 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1166 active.as_mut().expect("active stream generation").3 = false;
1167 return;
1168 }
1169 Err(error) => {
1170 drop(source);
1171 send_stream_failure(
1172 results,
1173 shutdown,
1174 info.id,
1175 run.generation,
1176 sequence,
1177 Some(logical_id),
1178 true,
1179 StreamFailure::Transform(error),
1180 );
1181 active.as_mut().expect("active stream generation").3 = false;
1182 run.cancel();
1183 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, false);
1184 return;
1185 }
1186 };
1187 let permit = if let Some(footprint) = footprint {
1188 let actual = footprint(&sample);
1189 let budget = run
1190 .byte_budget
1191 .as_ref()
1192 .expect("enabled stream byte accounting has a generation budget");
1193 let acquired = if ordered {
1194 let Some(sequence) = sequence else {
1195 drop(source);
1196 send_stream_failure(
1197 results,
1198 shutdown,
1199 info.id,
1200 run.generation,
1201 None,
1202 Some(logical_id),
1203 true,
1204 StreamFailure::Memory(
1205 M::protocol(
1206 None,
1207 "ordered byte-bounded stream record did not provide a sequence identifier".to_owned(),
1208 )
1209 .expect("enabled stream accounting has protocol failure storage"),
1210 ),
1211 );
1212 active.as_mut().expect("active stream generation").3 = false;
1213 run.cancel();
1214 send_stream_end::<S, F, I, M>(
1215 results,
1216 shutdown,
1217 info.id,
1218 run.generation,
1219 false,
1220 );
1221 return;
1222 };
1223 if last_sequence.is_some_and(|previous| sequence <= previous) {
1224 drop(source);
1225 send_stream_failure(
1226 results,
1227 shutdown,
1228 info.id,
1229 run.generation,
1230 Some(sequence),
1231 Some(logical_id),
1232 true,
1233 StreamFailure::Memory(
1234 M::protocol(
1235 Some(sequence),
1236 format!(
1237 "worker {} emitted non-increasing sequence {sequence}",
1238 info.id
1239 ),
1240 )
1241 .expect("enabled stream accounting has protocol failure storage"),
1242 ),
1243 );
1244 active.as_mut().expect("active stream generation").3 = false;
1245 run.cancel();
1246 send_stream_end::<S, F, I, M>(
1247 results,
1248 shutdown,
1249 info.id,
1250 run.generation,
1251 false,
1252 );
1253 return;
1254 }
1255 last_sequence = Some(sequence);
1256 if !send_stream_completion(
1257 results,
1258 shutdown,
1259 Some(&run.cancellation),
1260 StreamCompletion {
1261 worker: info.id,
1262 generation: run.generation,
1263 sequence: Some(sequence),
1264 logical_id: Some(logical_id),
1265 holds_credit: false,
1266 result: Ok(StreamMessage::Waiting(
1267 M::waiting(sequence)
1268 .expect("enabled stream accounting has waiter messages"),
1269 )),
1270 },
1271 ) {
1272 drop(source);
1273 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1274 active.as_mut().expect("active stream generation").3 = false;
1275 return;
1276 }
1277 budget.acquire_ordered(sequence, actual)
1278 } else {
1279 budget.acquire(actual)
1280 };
1281 match acquired {
1282 Ok(permit) => Some(permit),
1283 Err(BudgetError::Cancelled) => {
1284 drop(source);
1285 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1286 active.as_mut().expect("active stream generation").3 = false;
1287 return;
1288 }
1289 Err(BudgetError::Oversize { limit, actual }) => {
1290 drop(source);
1291 send_stream_failure(
1292 results,
1293 shutdown,
1294 info.id,
1295 run.generation,
1296 sequence,
1297 Some(logical_id),
1298 true,
1299 StreamFailure::Memory(
1300 M::memory_limit(limit, actual)
1301 .expect("enabled stream accounting has memory-limit storage"),
1302 ),
1303 );
1304 active.as_mut().expect("active stream generation").3 = false;
1305 run.cancel();
1306 send_stream_end::<S, F, I, M>(
1307 results,
1308 shutdown,
1309 info.id,
1310 run.generation,
1311 false,
1312 );
1313 return;
1314 }
1315 Err(BudgetError::SequenceAlreadyAdmitted) => {
1316 drop(source);
1317 send_stream_failure(
1318 results,
1319 shutdown,
1320 info.id,
1321 run.generation,
1322 sequence,
1323 Some(logical_id),
1324 true,
1325 StreamFailure::Memory(
1326 M::protocol(
1327 sequence,
1328 "sequence was already admitted to the byte budget".to_owned(),
1329 )
1330 .expect("enabled stream accounting has protocol failure storage"),
1331 ),
1332 );
1333 active.as_mut().expect("active stream generation").3 = false;
1334 run.cancel();
1335 send_stream_end::<S, F, I, M>(
1336 results,
1337 shutdown,
1338 info.id,
1339 run.generation,
1340 false,
1341 );
1342 return;
1343 }
1344 }
1345 } else {
1346 None
1347 };
1348 let sent = send_stream_completion(
1349 results,
1350 shutdown,
1351 Some(&run.cancellation),
1352 StreamCompletion {
1353 worker: info.id,
1354 generation: run.generation,
1355 sequence,
1356 logical_id: Some(logical_id),
1357 holds_credit: true,
1358 result: Ok(StreamMessage::Record(
1359 WorkerRecord {
1360 sequence: record.sequence,
1361 logical_id: record.logical_id,
1362 sample,
1363 },
1364 M::permit(permit),
1365 )),
1366 },
1367 );
1368 if sent {
1369 let active = active.as_mut().expect("active stream generation");
1370 active.1 = None;
1371 active.2 = None;
1372 active.3 = false;
1373 } else {
1374 drop(source);
1375 send_stream_end::<S, F, I, M>(results, shutdown, info.id, run.generation, true);
1376 active.as_mut().expect("active stream generation").3 = false;
1377 return;
1378 }
1379 }
1380}
1381
1382fn send_stream_end<S, F, I, M>(
1383 results: &Sender<StreamCompletionFor<S, F, I, M>>,
1384 shutdown: &CancellationToken,
1385 worker: usize,
1386 generation: u64,
1387 holds_credit: bool,
1388) where
1389 S: WorkerSourceFactory,
1390 F: TransformFactory<S::Sample>,
1391 I: WorkerInit,
1392 M: StreamMemoryPolicy,
1393{
1394 let _ = send_stream_completion(
1395 results,
1396 shutdown,
1397 None,
1398 StreamCompletion {
1399 worker,
1400 generation,
1401 sequence: None,
1402 logical_id: None,
1403 holds_credit,
1404 result: Ok(StreamMessage::End),
1405 },
1406 );
1407}
1408
1409#[allow(clippy::too_many_arguments)]
1410fn send_stream_failure<T, P, W, ME, SE, TE, FE, IE>(
1411 results: &StreamCompletionSender<T, P, W, ME, SE, TE, FE, IE>,
1412 shutdown: &CancellationToken,
1413 worker: usize,
1414 generation: u64,
1415 sequence: Option<u64>,
1416 logical_id: Option<u64>,
1417 holds_credit: bool,
1418 failure: StreamFailure<SE, TE, FE, IE, ME>,
1419) {
1420 let _ = send_stream_completion(
1421 results,
1422 shutdown,
1423 None,
1424 StreamCompletion {
1425 worker,
1426 generation,
1427 sequence,
1428 logical_id,
1429 holds_credit,
1430 result: Err(failure),
1431 },
1432 );
1433}
1434
1435fn send_stream_completion<T, P, W, ME, SE, TE, FE, IE>(
1436 results: &StreamCompletionSender<T, P, W, ME, SE, TE, FE, IE>,
1437 shutdown: &CancellationToken,
1438 cancellation: Option<&CancellationToken>,
1439 completion: StreamCompletion<T, P, W, ME, SE, TE, FE, IE>,
1440) -> bool {
1441 if let Some(cancellation) = cancellation {
1442 select! {
1443 send(results, completion) -> result => result.is_ok(),
1444 recv(cancellation.signal()) -> _ => false,
1445 recv(shutdown.signal()) -> _ => false,
1446 }
1447 } else {
1448 select! {
1449 send(results, completion) -> result => result.is_ok(),
1450 recv(shutdown.signal()) -> _ => false,
1451 }
1452 }
1453}
1454
1455#[derive(Clone, Copy)]
1456struct StreamConfiguration {
1457 workers: usize,
1458 batch_size: usize,
1459 drop_last: bool,
1460 prefetch_factor: usize,
1461 ordered: bool,
1462 persistent_workers: bool,
1463 timeout: Option<Duration>,
1464 loader_seed: u64,
1465 epoch: u64,
1466 rank: usize,
1467 prefetch_bytes: Option<NonZeroUsize>,
1468 pin_memory: StreamPinRequest,
1469}
1470
1471#[derive(Clone, Copy)]
1472enum StreamPinRequest {
1473 Disabled,
1474 Auto,
1475 Explicit(Device),
1476}
1477
1478impl Default for StreamConfiguration {
1479 fn default() -> Self {
1480 Self {
1481 workers: 1,
1482 batch_size: 1,
1483 drop_last: false,
1484 prefetch_factor: 2,
1485 ordered: true,
1486 persistent_workers: false,
1487 timeout: None,
1488 loader_seed: 0,
1489 epoch: 0,
1490 rank: 0,
1491 prefetch_bytes: None,
1492 pin_memory: StreamPinRequest::Disabled,
1493 }
1494 }
1495}
1496
1497pub struct StreamDataLoaderBuilder<
1575 S,
1576 C = DefaultCollator,
1577 F = IdentityTransformFactory,
1578 I = NoWorkerInit,
1579 M = MemoryDisabled,
1580 N = PinDisabled,
1581 K = crate::CheckpointDisabled,
1582> {
1583 factory: S,
1584 collator: C,
1585 transform_factory: F,
1586 worker_init: I,
1587 configuration: StreamConfiguration,
1588 checkpoint: K,
1589 states: PhantomData<(M, N)>,
1590}
1591
1592impl<S> StreamDataLoaderBuilder<S>
1593where
1594 S: WorkerSourceFactory,
1595{
1596 pub fn new(factory: S) -> Self {
1599 Self {
1600 factory,
1601 collator: DefaultCollator,
1602 transform_factory: IdentityTransformFactory,
1603 worker_init: NoWorkerInit,
1604 configuration: StreamConfiguration::default(),
1605 checkpoint: crate::CheckpointDisabled,
1606 states: PhantomData,
1607 }
1608 }
1609}
1610
1611impl<S, C, F, I, M, N, K> StreamDataLoaderBuilder<S, C, F, I, M, N, K> {
1612 pub fn workers(mut self, workers: usize) -> Self {
1614 self.configuration.workers = workers;
1615 self
1616 }
1617
1618 pub fn batch_size(mut self, batch_size: usize) -> Self {
1620 self.configuration.batch_size = batch_size;
1621 self
1622 }
1623
1624 pub fn drop_last(mut self, drop_last: bool) -> Self {
1626 self.configuration.drop_last = drop_last;
1627 self
1628 }
1629
1630 pub fn prefetch_factor(mut self, factor: usize) -> Self {
1635 self.configuration.prefetch_factor = factor;
1636 self
1637 }
1638
1639 pub fn prefetch_bytes(
1645 mut self,
1646 limit: NonZeroUsize,
1647 ) -> StreamDataLoaderBuilder<S, C, F, I, MemoryEnabled, N, K> {
1648 self.configuration.prefetch_bytes = Some(limit);
1649 StreamDataLoaderBuilder {
1650 factory: self.factory,
1651 collator: self.collator,
1652 transform_factory: self.transform_factory,
1653 worker_init: self.worker_init,
1654 configuration: self.configuration,
1655 checkpoint: self.checkpoint,
1656 states: PhantomData,
1657 }
1658 }
1659
1660 pub fn pin_memory(mut self) -> StreamDataLoaderBuilder<S, C, F, I, M, PinEnabled<Auto>, K> {
1662 self.configuration.pin_memory = StreamPinRequest::Auto;
1663 StreamDataLoaderBuilder {
1664 factory: self.factory,
1665 collator: self.collator,
1666 transform_factory: self.transform_factory,
1667 worker_init: self.worker_init,
1668 configuration: self.configuration,
1669 checkpoint: self.checkpoint,
1670 states: PhantomData,
1671 }
1672 }
1673
1674 pub fn pin_memory_for(
1676 mut self,
1677 device: Device,
1678 ) -> StreamDataLoaderBuilder<S, C, F, I, M, PinEnabled<Explicit>, K> {
1679 self.configuration.pin_memory = StreamPinRequest::Explicit(device);
1680 StreamDataLoaderBuilder {
1681 factory: self.factory,
1682 collator: self.collator,
1683 transform_factory: self.transform_factory,
1684 worker_init: self.worker_init,
1685 configuration: self.configuration,
1686 checkpoint: self.checkpoint,
1687 states: PhantomData,
1688 }
1689 }
1690
1691 pub fn ordered(mut self, ordered: bool) -> Self {
1693 self.configuration.ordered = ordered;
1694 self
1695 }
1696
1697 pub fn in_order(self, in_order: bool) -> Self {
1699 self.ordered(in_order)
1700 }
1701
1702 pub fn seed(mut self, seed: u64) -> Self {
1704 self.configuration.loader_seed = seed;
1705 self
1706 }
1707
1708 pub fn epoch(mut self, epoch: u64) -> Self {
1710 self.configuration.epoch = epoch;
1711 self
1712 }
1713
1714 pub fn rank(mut self, rank: usize) -> Self {
1716 self.configuration.rank = rank;
1717 self
1718 }
1719
1720 pub fn timeout(mut self, timeout: Duration) -> Self {
1722 self.configuration.timeout = (!timeout.is_zero()).then_some(timeout);
1723 self
1724 }
1725
1726 pub fn persistent_workers(mut self, persistent: bool) -> Self {
1728 self.configuration.persistent_workers = persistent;
1729 self
1730 }
1731
1732 pub fn collate<C2>(self, collator: C2) -> StreamDataLoaderBuilder<S, C2, F, I, M, N, K> {
1734 StreamDataLoaderBuilder {
1735 factory: self.factory,
1736 collator,
1737 transform_factory: self.transform_factory,
1738 worker_init: self.worker_init,
1739 configuration: self.configuration,
1740 checkpoint: self.checkpoint,
1741 states: PhantomData,
1742 }
1743 }
1744
1745 pub fn transform<T>(
1747 self,
1748 transform: T,
1749 ) -> StreamDataLoaderBuilder<S, C, CloneTransformFactory<T>, I, M, N, K> {
1750 StreamDataLoaderBuilder {
1751 factory: self.factory,
1752 collator: self.collator,
1753 transform_factory: CloneTransformFactory::new(transform),
1754 worker_init: self.worker_init,
1755 configuration: self.configuration,
1756 checkpoint: self.checkpoint,
1757 states: PhantomData,
1758 }
1759 }
1760
1761 pub fn transform_factory<F2>(
1763 self,
1764 factory: F2,
1765 ) -> StreamDataLoaderBuilder<S, C, F2, I, M, N, K> {
1766 StreamDataLoaderBuilder {
1767 factory: self.factory,
1768 collator: self.collator,
1769 transform_factory: factory,
1770 worker_init: self.worker_init,
1771 configuration: self.configuration,
1772 checkpoint: self.checkpoint,
1773 states: PhantomData,
1774 }
1775 }
1776
1777 pub fn worker_init<I2>(self, worker_init: I2) -> StreamDataLoaderBuilder<S, C, F, I2, M, N, K> {
1779 StreamDataLoaderBuilder {
1780 factory: self.factory,
1781 collator: self.collator,
1782 transform_factory: self.transform_factory,
1783 worker_init,
1784 configuration: self.configuration,
1785 checkpoint: self.checkpoint,
1786 states: PhantomData,
1787 }
1788 }
1789
1790 fn build_inner(
1792 self,
1793 footprint: Option<StreamFootprint<S, F>>,
1794 ) -> Result<StreamDataLoader<S, C, F, I, M, N>>
1795 where
1796 S: WorkerSourceFactory,
1797 F: TransformFactory<S::Sample>,
1798 C: Collate<TransformOutput<S, F>>,
1799 I: WorkerInit,
1800 M: StreamMemoryPolicy,
1801 {
1802 let pin_memory_status = resolve_stream_pin(self.configuration.pin_memory)?;
1803 if self.configuration.workers == 0 {
1804 return Err(invalid_configuration(
1805 "workers",
1806 "stream workers must be positive; use batches or batches_with_collate for an ordinary iterator",
1807 ));
1808 }
1809 if self.configuration.batch_size == 0 {
1810 return Err(invalid_configuration(
1811 "batch_size",
1812 "must be greater than zero",
1813 ));
1814 }
1815 if self.configuration.prefetch_factor == 0 {
1816 return Err(invalid_configuration(
1817 "prefetch_factor",
1818 "must be greater than zero",
1819 ));
1820 }
1821 if self
1822 .configuration
1823 .timeout
1824 .is_some_and(|timeout| !Deadline::can_represent(timeout))
1825 {
1826 return Err(invalid_configuration(
1827 "timeout",
1828 "exceeds the platform monotonic clock range",
1829 ));
1830 }
1831 let outstanding_capacity = self
1832 .configuration
1833 .workers
1834 .checked_mul(self.configuration.prefetch_factor)
1835 .ok_or_else(|| {
1836 invalid_configuration(
1837 "prefetch_factor",
1838 "workers multiplied by prefetch_factor exceeds usize",
1839 )
1840 })?;
1841 validate_stream_capacity::<S, F, I, M>(
1842 self.configuration.workers,
1843 self.configuration.prefetch_factor,
1844 outstanding_capacity,
1845 self.configuration.batch_size,
1846 usize::from(self.configuration.ordered) * outstanding_capacity,
1847 )?;
1848 let mut ordered_reassembly = Vec::new();
1849 if self.configuration.ordered {
1850 ordered_reassembly
1851 .try_reserve_exact(outstanding_capacity)
1852 .map_err(|error| {
1853 capacity_error(format!(
1854 "ordered stream reassembly capacity is unavailable: {error}"
1855 ))
1856 })?;
1857 }
1858 validate_stream_capacity::<S, F, I, M>(
1859 self.configuration.workers,
1860 self.configuration.prefetch_factor,
1861 outstanding_capacity,
1862 self.configuration.batch_size,
1863 ordered_reassembly.capacity(),
1864 )?;
1865 let exact_len = self.factory.exact_len();
1866 Ok(StreamDataLoader {
1867 factory: Arc::new(self.factory),
1868 collator: self.collator,
1869 transform_factory: Arc::new(self.transform_factory),
1870 worker_init: Arc::new(self.worker_init),
1871 configuration: self.configuration,
1872 outstanding_capacity,
1873 exact_len,
1874 next_generation: 0,
1875 persistent_pool: None,
1876 ordered_reassembly: Mutex::new(ordered_reassembly),
1877 footprint,
1878 pin_memory_status,
1879 policies: PhantomData,
1880 })
1881 }
1882}
1883
1884fn stream_resident_bytes<T: MemoryFootprint>(value: &T) -> usize {
1885 value.resident_bytes()
1886}
1887
1888fn resolve_stream_pin(request: StreamPinRequest) -> Result<PinMemoryStatus> {
1889 match request {
1890 StreamPinRequest::Disabled => Ok(PinMemoryStatus::Disabled),
1891 StreamPinRequest::Auto => {
1892 let capabilities = available_devices();
1893 if capabilities.cuda && capabilities.cuda_device_count > 0 {
1894 Ok(PinMemoryStatus::Enabled(Device::Cuda(0)))
1895 } else {
1896 Ok(PinMemoryStatus::DisabledNoAccelerator)
1897 }
1898 }
1899 StreamPinRequest::Explicit(Device::Cuda(index)) => {
1900 let capabilities = available_devices();
1901 if capabilities.cuda && index < capabilities.cuda_device_count {
1902 Ok(PinMemoryStatus::Enabled(Device::Cuda(index)))
1903 } else {
1904 Err(invalid_configuration(
1905 "pin_memory",
1906 format!(
1907 "CUDA device {index} was requested, but the linked runtime exposes {} available CUDA device(s)",
1908 capabilities.cuda_device_count
1909 ),
1910 ))
1911 }
1912 }
1913 StreamPinRequest::Explicit(device) => Err(invalid_configuration(
1914 "pin_memory",
1915 format!("only an available CUDA device can back pinned host memory, got {device:?}"),
1916 )),
1917 }
1918}
1919
1920impl<S, C, F, I> StreamDataLoaderBuilder<S, C, F, I, MemoryDisabled, PinDisabled> {
1921 pub fn build(self) -> Result<StreamDataLoader<S, C, F, I, MemoryDisabled, PinDisabled>>
1923 where
1924 S: WorkerSourceFactory,
1925 F: TransformFactory<S::Sample>,
1926 C: Collate<TransformOutput<S, F>>,
1927 I: WorkerInit,
1928 {
1929 self.build_inner(None)
1930 }
1931}
1932
1933impl<S, C, F, I> StreamDataLoaderBuilder<S, C, F, I, MemoryEnabled, PinDisabled> {
1934 pub fn build(self) -> Result<StreamDataLoader<S, C, F, I, MemoryEnabled, PinDisabled>>
1936 where
1937 S: WorkerSourceFactory,
1938 F: TransformFactory<S::Sample>,
1939 TransformOutput<S, F>: MemoryFootprint,
1940 C: Collate<TransformOutput<S, F>>,
1941 I: WorkerInit,
1942 {
1943 self.build_inner(Some(stream_resident_bytes::<TransformOutput<S, F>>))
1944 }
1945}
1946
1947impl<S, C, F, I, Q> StreamDataLoaderBuilder<S, C, F, I, MemoryDisabled, PinEnabled<Q>> {
1948 pub fn build(self) -> Result<StreamDataLoader<S, C, F, I, MemoryDisabled, PinEnabled<Q>>>
1950 where
1951 S: WorkerSourceFactory,
1952 F: TransformFactory<S::Sample>,
1953 C: Collate<TransformOutput<S, F>>,
1954 C::Batch: PinMemory,
1955 I: WorkerInit,
1956 {
1957 self.build_inner(None)
1958 }
1959}
1960
1961impl<S, C, F, I, Q> StreamDataLoaderBuilder<S, C, F, I, MemoryEnabled, PinEnabled<Q>> {
1962 pub fn build(self) -> Result<StreamDataLoader<S, C, F, I, MemoryEnabled, PinEnabled<Q>>>
1964 where
1965 S: WorkerSourceFactory,
1966 F: TransformFactory<S::Sample>,
1967 TransformOutput<S, F>: MemoryFootprint,
1968 C: Collate<TransformOutput<S, F>>,
1969 C::Batch: PinMemory,
1970 I: WorkerInit,
1971 {
1972 self.build_inner(Some(stream_resident_bytes::<TransformOutput<S, F>>))
1973 }
1974}
1975
1976#[allow(private_bounds)]
1988pub struct StreamDataLoader<
1989 S,
1990 C,
1991 F = IdentityTransformFactory,
1992 I = NoWorkerInit,
1993 M = MemoryDisabled,
1994 N = PinDisabled,
1995> where
1996 S: WorkerSourceFactory,
1997 F: TransformFactory<S::Sample>,
1998 C: Collate<TransformOutput<S, F>>,
1999 I: WorkerInit,
2000 M: StreamMemoryPolicy,
2001{
2002 factory: Arc<S>,
2003 collator: C,
2004 transform_factory: Arc<F>,
2005 worker_init: Arc<I>,
2006 configuration: StreamConfiguration,
2007 outstanding_capacity: usize,
2008 exact_len: Option<usize>,
2009 next_generation: u64,
2010 persistent_pool: Option<StreamWorkerPool<S, F, I, M>>,
2011 ordered_reassembly: RetainedReassembly<TransformOutput<S, F>, M::Permit>,
2014 footprint: Option<StreamFootprint<S, F>>,
2015 pin_memory_status: PinMemoryStatus,
2016 policies: PhantomData<(M, N)>,
2017}
2018
2019#[allow(private_bounds)]
2020impl<S, C, F, I, M, N> StreamDataLoader<S, C, F, I, M, N>
2021where
2022 S: WorkerSourceFactory,
2023 F: TransformFactory<S::Sample>,
2024 C: Collate<TransformOutput<S, F>>,
2025 I: WorkerInit,
2026 M: StreamMemoryPolicy,
2027{
2028 pub fn len(&self) -> Option<usize> {
2030 self.exact_len.map(|length| {
2031 let complete = length / self.configuration.batch_size;
2032 complete
2033 + usize::from(
2034 !self.configuration.drop_last
2035 && !length.is_multiple_of(self.configuration.batch_size),
2036 )
2037 })
2038 }
2039
2040 pub fn is_empty(&self) -> Option<bool> {
2042 self.len().map(|length| length == 0)
2043 }
2044
2045 pub fn workers(&self) -> usize {
2047 self.configuration.workers
2048 }
2049
2050 pub fn effective_prefetch_factor(&self) -> NonZeroUsize {
2052 NonZeroUsize::new(self.configuration.prefetch_factor)
2053 .expect("validated prefetch factor is nonzero")
2054 }
2055
2056 pub fn effective_prefetch_bytes(&self) -> Option<NonZeroUsize> {
2058 self.configuration.prefetch_bytes
2059 }
2060
2061 pub fn pin_memory_enabled(&self) -> bool {
2063 !matches!(self.pin_memory_status, PinMemoryStatus::Disabled)
2064 }
2065
2066 pub fn pin_memory_status(&self) -> PinMemoryStatus {
2068 self.pin_memory_status
2069 }
2070
2071 pub fn is_ordered(&self) -> bool {
2073 self.configuration.ordered
2074 }
2075
2076 pub fn epoch(&self) -> u64 {
2078 self.configuration.epoch
2079 }
2080
2081 pub fn set_epoch(&mut self, epoch: u64) {
2083 self.configuration.epoch = epoch;
2084 }
2085}
2086
2087struct BufferedRecord<T, P> {
2088 worker: usize,
2089 record: WorkerRecord<T>,
2090 permit: P,
2091}
2092
2093enum StreamIteratorPool<'a, S, F, I, M>
2094where
2095 S: WorkerSourceFactory,
2096 F: TransformFactory<S::Sample>,
2097 I: WorkerInit,
2098 M: StreamMemoryPolicy,
2099{
2100 Owned(StreamWorkerPool<S, F, I, M>),
2101 Persistent(&'a mut StreamWorkerPool<S, F, I, M>),
2102}
2103
2104impl<S, F, I, M> StreamIteratorPool<'_, S, F, I, M>
2105where
2106 S: WorkerSourceFactory,
2107 F: TransformFactory<S::Sample>,
2108 I: WorkerInit,
2109 M: StreamMemoryPolicy,
2110{
2111 fn pool(&self) -> &StreamWorkerPool<S, F, I, M> {
2112 match self {
2113 Self::Owned(pool) => pool,
2114 Self::Persistent(pool) => pool,
2115 }
2116 }
2117
2118 fn pool_mut(&mut self) -> &mut StreamWorkerPool<S, F, I, M> {
2119 match self {
2120 Self::Owned(pool) => pool,
2121 Self::Persistent(pool) => pool,
2122 }
2123 }
2124
2125 fn is_persistent(&self) -> bool {
2126 matches!(self, Self::Persistent(_))
2127 }
2128}
2129
2130#[allow(private_bounds)]
2131impl<S, C, F, I, M, N> StreamDataLoader<S, C, F, I, M, N>
2132where
2133 S: WorkerSourceFactory,
2134 F: TransformFactory<S::Sample> + Send + Sync + 'static,
2135 F::Transform: Send + 'static,
2136 TransformOutput<S, F>: Send + 'static,
2137 TransformFailure<S, F>: Send + 'static,
2138 F::Error: Send + 'static,
2139 C: Collate<TransformOutput<S, F>>,
2140 I: WorkerInit + Send + Sync + 'static,
2141 I::Error: Send + 'static,
2142 M: StreamMemoryPolicy + Send + 'static,
2143 N: StreamPinPolicy<S, C, F>,
2144{
2145 pub fn iter(&mut self) -> StreamLoaderIter<'_, S, C, F, I, M, N> {
2147 let ordered_reassembly = match self.ordered_reassembly.get_mut() {
2148 Ok(reassembly) => reassembly,
2149 Err(poisoned) => poisoned.into_inner(),
2150 };
2151 ordered_reassembly.clear();
2152 let reassembly_allocation_slots = ordered_reassembly.capacity();
2153 let generation = self.next_generation;
2154 let run_context = WorkerRunContext::new(
2155 generation,
2156 self.configuration.loader_seed,
2157 self.configuration.epoch,
2158 )
2159 .with_byte_budget(
2160 self.configuration
2161 .prefetch_bytes
2162 .map(|limit| ByteBudget::new(limit.get())),
2163 );
2164 let mut pending_error = None;
2165 if let Some(next) = generation.checked_add(1) {
2166 self.next_generation = next;
2167 } else {
2168 pending_error = Some(LoaderError::Configuration(invalid_configuration(
2169 "workers",
2170 "stream iterator generation overflowed",
2171 )));
2172 }
2173
2174 let mut partial = Vec::new();
2175 if partial
2176 .try_reserve_exact(self.configuration.batch_size)
2177 .is_err()
2178 {
2179 pending_error = Some(LoaderError::Configuration(invalid_configuration(
2180 "batch_size",
2181 "stream batch allocation is unavailable",
2182 )));
2183 }
2184
2185 let front_waiters = match M::new_waiters(self.configuration.workers) {
2186 Ok(waiters) => waiters,
2187 Err(error) => {
2188 pending_error = Some(LoaderError::Configuration(error));
2189 M::new_waiters(0).expect("zero waiter state is always allocatable")
2190 }
2191 };
2192
2193 let mut pool = None;
2194 if pending_error.is_none() {
2195 if self.configuration.persistent_workers {
2196 if self.persistent_pool.is_none() {
2197 match StreamWorkerPool::new(
2198 Arc::clone(&self.factory),
2199 Arc::clone(&self.transform_factory),
2200 Arc::clone(&self.worker_init),
2201 self.configuration,
2202 generation,
2203 self.outstanding_capacity,
2204 reassembly_allocation_slots,
2205 self.footprint,
2206 ) {
2207 Ok(created) => self.persistent_pool = Some(created),
2208 Err(error) => pending_error = Some(LoaderError::Configuration(error)),
2209 }
2210 }
2211 if pending_error.is_none() {
2212 let persistent = self
2213 .persistent_pool
2214 .as_mut()
2215 .expect("persistent stream pool was created");
2216 if persistent.start_generation(run_context.clone()).is_err() {
2217 pending_error = Some(LoaderError::ChannelClosed { batch: 0 });
2218 } else {
2219 pool = Some(StreamIteratorPool::Persistent(persistent));
2220 }
2221 }
2222 } else {
2223 match StreamWorkerPool::new(
2224 Arc::clone(&self.factory),
2225 Arc::clone(&self.transform_factory),
2226 Arc::clone(&self.worker_init),
2227 self.configuration,
2228 generation,
2229 self.outstanding_capacity,
2230 reassembly_allocation_slots,
2231 self.footprint,
2232 ) {
2233 Ok(mut created) => {
2234 if created.start_generation(run_context.clone()).is_err() {
2235 pending_error = Some(LoaderError::ChannelClosed { batch: 0 });
2236 } else {
2237 pool = Some(StreamIteratorPool::Owned(created));
2238 }
2239 }
2240 Err(error) => pending_error = Some(LoaderError::Configuration(error)),
2241 }
2242 }
2243 }
2244
2245 StreamLoaderIter {
2246 collator: &mut self.collator,
2247 pool,
2248 run_context,
2249 generation,
2250 ordered: self.configuration.ordered,
2251 timeout: self.configuration.timeout,
2252 batch_size: self.configuration.batch_size,
2253 drop_last: self.configuration.drop_last,
2254 expected_records: self.exact_len,
2255 reassembly_capacity: self.outstanding_capacity,
2256 completed: ordered_reassembly,
2257 partial,
2258 next_sequence: 0,
2259 next_batch: 0,
2260 pending_error,
2261 source_complete: false,
2262 exhausted: false,
2263 front_waiters,
2264 pin_memory_status: self.pin_memory_status,
2265 marker: PhantomData,
2266 }
2267 }
2268}
2269
2270#[allow(private_bounds)]
2277pub struct StreamLoaderIter<'a, S, C, F, I, M = MemoryDisabled, N = PinDisabled>
2278where
2279 S: WorkerSourceFactory,
2280 F: TransformFactory<S::Sample>,
2281 C: Collate<TransformOutput<S, F>>,
2282 I: WorkerInit,
2283 M: StreamMemoryPolicy,
2284{
2285 collator: &'a mut C,
2286 pool: Option<StreamIteratorPool<'a, S, F, I, M>>,
2287 run_context: WorkerRunContext,
2288 generation: u64,
2289 ordered: bool,
2290 timeout: Option<Duration>,
2291 batch_size: usize,
2292 drop_last: bool,
2293 expected_records: Option<usize>,
2294 reassembly_capacity: usize,
2295 completed: &'a mut Vec<ReassemblyEntry<TransformOutput<S, F>, M::Permit>>,
2296 partial: Vec<TransformOutput<S, F>>,
2297 next_sequence: u64,
2298 next_batch: u64,
2299 pending_error: Option<StreamLoaderError<S, C, F, I>>,
2300 source_complete: bool,
2301 exhausted: bool,
2302 front_waiters: M::Waiters,
2303 pin_memory_status: PinMemoryStatus,
2304 marker: PhantomData<&'a mut (S, C, F, I, M, N)>,
2305}
2306
2307#[allow(private_bounds)]
2308impl<S, C, F, I, M, N> StreamLoaderIter<'_, S, C, F, I, M, N>
2309where
2310 S: WorkerSourceFactory,
2311 F: TransformFactory<S::Sample> + Send + Sync + 'static,
2312 F::Transform: Send + 'static,
2313 TransformOutput<S, F>: Send + 'static,
2314 TransformFailure<S, F>: Send + 'static,
2315 F::Error: Send + 'static,
2316 C: Collate<TransformOutput<S, F>>,
2317 I: WorkerInit + Send + Sync + 'static,
2318 I::Error: Send + 'static,
2319 M: StreamMemoryPolicy + Send + 'static,
2320 N: StreamPinPolicy<S, C, F>,
2321{
2322 fn return_credit(&mut self, worker: usize) -> std::result::Result<(), ()> {
2323 self.pool.as_ref().ok_or(())?.pool().return_credit(worker)
2324 }
2325
2326 fn disarm_deadline(&self) {
2327 self.run_context.deadline.disarm();
2328 }
2329
2330 fn set_front_waiter(&mut self, worker: usize, sequence: Option<u64>) {
2331 if let Some(waiters) = M::waiters_mut(&mut self.front_waiters) {
2332 waiters[worker] = sequence;
2333 }
2334 }
2335
2336 fn close(&mut self, poisoned: bool) {
2337 self.exhausted = true;
2338 self.run_context.cancel();
2339 let mut credit_failed = false;
2340 while let Some((_, buffered)) = self.completed.pop() {
2341 credit_failed |= self.return_credit(buffered.worker).is_err();
2342 }
2343 if let Some(mut pool) = self.pool.take() {
2344 let persistent = pool.is_persistent();
2345 let quiesce_failed = persistent && pool.pool_mut().quiesce(self.generation).is_err();
2346 if poisoned || credit_failed || quiesce_failed {
2347 pool.pool_mut().poison();
2348 }
2349 }
2350 self.partial.clear();
2351 }
2352
2353 fn protocol_error(
2354 &mut self,
2355 sequence: Option<u64>,
2356 reason: impl Into<String>,
2357 ) -> StreamLoaderError<S, C, F, I> {
2358 self.disarm_deadline();
2359 self.close(false);
2360 LoaderError::StreamProtocol {
2361 sequence,
2362 reason: reason.into(),
2363 }
2364 }
2365
2366 fn consume(
2367 &mut self,
2368 buffered: BufferedRecord<TransformOutput<S, F>, M::Permit>,
2369 ) -> std::result::Result<bool, StreamLoaderError<S, C, F, I>> {
2370 let BufferedRecord {
2371 worker,
2372 record,
2373 permit,
2374 } = buffered;
2375 let next_sequence = if self.ordered {
2376 let Some(next) = record
2377 .sequence
2378 .expect("ordered records are validated")
2379 .checked_next()
2380 else {
2381 self.run_context.cancel();
2382 if self.return_credit(worker).is_err() {
2383 self.close(true);
2384 return Err(LoaderError::ChannelClosed {
2385 batch: self.next_batch,
2386 });
2387 }
2388 return Err(self.protocol_error(
2389 Some(self.next_sequence),
2390 "stream sequence identifier overflowed",
2391 ));
2392 };
2393 Some(next.get())
2394 } else {
2395 None
2396 };
2397 if self.return_credit(worker).is_err() {
2398 self.close(true);
2399 return Err(LoaderError::ChannelClosed {
2400 batch: self.next_batch,
2401 });
2402 }
2403 if let Some(next_sequence) = next_sequence {
2404 self.next_sequence = next_sequence;
2405 }
2406 drop(permit);
2407 self.partial.push(record.sample);
2408 Ok(self.partial.len() == self.batch_size)
2409 }
2410
2411 fn finish_batch(&mut self) -> std::result::Result<C::Batch, StreamLoaderError<S, C, F, I>> {
2412 let mut replacement = Vec::new();
2413 if replacement.try_reserve_exact(self.batch_size).is_err() {
2414 self.close(false);
2415 return Err(LoaderError::Configuration(invalid_configuration(
2416 "batch_size",
2417 "stream batch allocation is unavailable",
2418 )));
2419 }
2420 let samples = std::mem::replace(&mut self.partial, replacement);
2421 let batch = self.next_batch;
2422 let result = match catch_unwind(AssertUnwindSafe(|| self.collator.collate(samples))) {
2423 Ok(result) => result.map_err(|source| LoaderError::Pipeline {
2424 batch: Some(batch),
2425 worker: None,
2426 source: PipelineError::Collate(source),
2427 }),
2428 Err(_) => Err(LoaderError::CoordinatorPanic {
2429 stage: "stream collation",
2430 batch: Some(batch),
2431 }),
2432 }
2433 .and_then(|value| match self.pin_memory_status {
2434 PinMemoryStatus::Enabled(device) => {
2435 N::pin(value, device).map_err(|source| LoaderError::PinMemory { batch, source })
2436 }
2437 PinMemoryStatus::Disabled | PinMemoryStatus::DisabledNoAccelerator => Ok(value),
2438 });
2439 match result {
2440 Ok(batch) => match self.next_batch.checked_add(1) {
2441 Some(next) => {
2442 self.next_batch = next;
2443 self.disarm_deadline();
2444 Ok(batch)
2445 }
2446 None => {
2447 self.close(false);
2448 Err(LoaderError::Configuration(invalid_configuration(
2449 "batch_size",
2450 "stream batch sequence overflowed",
2451 )))
2452 }
2453 },
2454 Err(error) => {
2455 self.disarm_deadline();
2456 self.close(false);
2457 Err(error)
2458 }
2459 }
2460 }
2461
2462 fn map_failure(
2463 &mut self,
2464 worker: usize,
2465 sequence: Option<u64>,
2466 logical_id: Option<u64>,
2467 failure: StreamFailureFor<S, F, I, M>,
2468 ) -> StreamLoaderError<S, C, F, I> {
2469 let batch = Some(self.next_batch);
2470 match failure {
2471 StreamFailure::Source(source) => LoaderError::StreamPipeline {
2472 batch,
2473 worker,
2474 sequence,
2475 logical_id,
2476 source: PipelineError::Source(source),
2477 },
2478 StreamFailure::Transform(source) => LoaderError::StreamPipeline {
2479 batch,
2480 worker,
2481 sequence,
2482 logical_id,
2483 source: PipelineError::Transform(source),
2484 },
2485 StreamFailure::TransformInit(source) => LoaderError::StreamPipeline {
2486 batch: None,
2487 worker,
2488 sequence: None,
2489 logical_id: None,
2490 source: PipelineError::TransformInit(source),
2491 },
2492 StreamFailure::WorkerInit(source) => LoaderError::StreamPipeline {
2493 batch: None,
2494 worker,
2495 sequence: None,
2496 logical_id: None,
2497 source: PipelineError::WorkerInit(source),
2498 },
2499 StreamFailure::Memory(failure) => M::map_failure(failure, worker, sequence, logical_id),
2500 StreamFailure::Panic => LoaderError::StreamWorkerPanic {
2501 worker,
2502 batch,
2503 sequence,
2504 logical_id,
2505 },
2506 }
2507 }
2508}
2509
2510impl<S, C, F, I, M, N> Iterator for StreamLoaderIter<'_, S, C, F, I, M, N>
2511where
2512 S: WorkerSourceFactory,
2513 F: TransformFactory<S::Sample> + Send + Sync + 'static,
2514 F::Transform: Send + 'static,
2515 TransformOutput<S, F>: Send + 'static,
2516 TransformFailure<S, F>: Send + 'static,
2517 F::Error: Send + 'static,
2518 C: Collate<TransformOutput<S, F>>,
2519 I: WorkerInit + Send + Sync + 'static,
2520 I::Error: Send + 'static,
2521 M: StreamMemoryPolicy + Send + 'static,
2522 N: StreamPinPolicy<S, C, F>,
2523{
2524 type Item = std::result::Result<C::Batch, StreamLoaderError<S, C, F, I>>;
2525
2526 fn next(&mut self) -> Option<Self::Item> {
2527 if self.exhausted {
2528 return None;
2529 }
2530 if let Some(error) = self.pending_error.take() {
2531 self.close(false);
2532 return Some(Err(error));
2533 }
2534 if self.source_complete {
2535 self.close(false);
2536 return None;
2537 }
2538 let mut deadline_armed = false;
2539 loop {
2540 if self.ordered
2541 && let Some(index) = self
2542 .completed
2543 .iter()
2544 .position(|(sequence, _)| *sequence == self.next_sequence)
2545 {
2546 let (_, buffered) = self.completed.swap_remove(index);
2547 match self.consume(buffered) {
2548 Ok(true) => return Some(self.finish_batch()),
2549 Ok(false) => continue,
2550 Err(error) => return Some(Err(error)),
2551 }
2552 }
2553
2554 if self.ordered && self.completed.len() == self.reassembly_capacity {
2555 let expected = self.next_sequence;
2556 let found = self
2557 .completed
2558 .iter()
2559 .map(|(sequence, _)| *sequence)
2560 .min()
2561 .expect("a full ordered reassembly window is nonempty");
2562 return Some(Err(self.protocol_error(
2563 Some(expected),
2564 format!(
2565 "missing sequence {expected}; bounded reassembly window is full before buffered sequence {found}"
2566 ),
2567 )));
2568 }
2569
2570 if self.ordered
2571 && self.run_context.byte_budget.is_some()
2572 && let Some(pool) = self.pool.as_ref()
2573 && let Some(front_waiters) = M::waiters(&self.front_waiters)
2574 && front_waiters
2575 .iter()
2576 .enumerate()
2577 .any(|(worker, _)| !pool.pool().terminal.get(worker).copied().unwrap_or(false))
2578 && front_waiters.iter().enumerate().all(|(worker, sequence)| {
2579 pool.pool().terminal.get(worker).copied().unwrap_or(false)
2580 || sequence.is_some_and(|sequence| sequence > self.next_sequence)
2581 })
2582 {
2583 let expected = self.next_sequence;
2584 let found = M::waiters(&self.front_waiters)
2585 .expect("enabled byte accounting has waiter state")
2586 .iter()
2587 .flatten()
2588 .copied()
2589 .min()
2590 .expect("a nonterminal byte-budget worker has a front waiter");
2591 return Some(Err(self.protocol_error(
2592 Some(expected),
2593 format!(
2594 "missing sequence {expected}; all active byte-budget workers are waiting at sequence {found} or later"
2595 ),
2596 )));
2597 }
2598
2599 if self
2600 .pool
2601 .as_ref()
2602 .is_some_and(|pool| pool.pool().all_terminal())
2603 {
2604 if self.ordered
2605 && let Some(found) = self.completed.iter().map(|(sequence, _)| *sequence).min()
2606 {
2607 let expected = self.next_sequence;
2608 return Some(Err(self.protocol_error(
2609 Some(expected),
2610 format!("missing sequence {expected} before buffered sequence {found}"),
2611 )));
2612 }
2613 if self.ordered
2614 && let Some(expected) = self.expected_records
2615 && self.next_sequence != expected as u64
2616 {
2617 let missing = self.next_sequence.min(expected as u64);
2618 return Some(Err(self.protocol_error(
2619 Some(missing),
2620 format!(
2621 "stream ended after {} ordered records, expected {expected}",
2622 self.next_sequence
2623 ),
2624 )));
2625 }
2626 self.source_complete = true;
2627 if self.partial.is_empty() || self.drop_last {
2628 self.close(false);
2629 return None;
2630 }
2631 return Some(self.finish_batch());
2632 }
2633
2634 if !deadline_armed && let Some(timeout) = self.timeout {
2635 self.run_context.deadline.arm(timeout);
2636 deadline_armed = true;
2637 }
2638 let remaining = self
2639 .timeout
2640 .and_then(|_| self.run_context.deadline.remaining());
2641 let received = self
2642 .pool
2643 .as_ref()
2644 .expect("active stream iterator has a pool")
2645 .pool()
2646 .receive(&self.run_context, remaining);
2647 let completion = match received {
2648 StreamReceive::Completion(completion) => completion,
2649 StreamReceive::Timeout => {
2650 let batch = self.next_batch;
2651 self.run_context.cancel();
2652 self.disarm_deadline();
2653 self.close(false);
2654 return Some(Err(LoaderError::Timeout { batch }));
2655 }
2656 StreamReceive::Cancelled => {
2657 self.disarm_deadline();
2658 self.close(false);
2659 return Some(Err(LoaderError::Cancelled));
2660 }
2661 StreamReceive::Closed => {
2662 let batch = self.next_batch;
2663 self.disarm_deadline();
2664 self.close(true);
2665 return Some(Err(LoaderError::ChannelClosed { batch }));
2666 }
2667 };
2668 if completion.generation != self.generation {
2669 if completion.holds_credit && self.return_credit(completion.worker).is_err() {
2670 self.close(true);
2671 return Some(Err(LoaderError::ChannelClosed {
2672 batch: self.next_batch,
2673 }));
2674 }
2675 continue;
2676 }
2677
2678 match completion.result {
2679 Err(failure) => {
2680 self.set_front_waiter(completion.worker, None);
2681 self.run_context.cancel();
2682 let fatal = matches!(
2683 &failure,
2684 StreamFailure::TransformInit(_)
2685 | StreamFailure::WorkerInit(_)
2686 | StreamFailure::Panic
2687 );
2688 if self
2689 .pool
2690 .as_mut()
2691 .expect("stream pool")
2692 .pool_mut()
2693 .account_parts(completion.worker, completion.holds_credit, fatal, fatal)
2694 .is_err()
2695 {
2696 self.close(true);
2697 return Some(Err(LoaderError::ChannelClosed {
2698 batch: self.next_batch,
2699 }));
2700 }
2701 let error = self.map_failure(
2702 completion.worker,
2703 completion.sequence,
2704 completion.logical_id,
2705 failure,
2706 );
2707 self.disarm_deadline();
2708 self.close(fatal);
2709 return Some(Err(error));
2710 }
2711 Ok(StreamMessage::End) => {
2712 self.set_front_waiter(completion.worker, None);
2713 if self
2714 .pool
2715 .as_mut()
2716 .expect("stream pool")
2717 .pool_mut()
2718 .account_parts(completion.worker, completion.holds_credit, true, false)
2719 .is_err()
2720 {
2721 self.close(true);
2722 return Some(Err(LoaderError::ChannelClosed {
2723 batch: self.next_batch,
2724 }));
2725 }
2726 }
2727 Ok(StreamMessage::Waiting(waiting)) => {
2728 self.set_front_waiter(completion.worker, Some(M::waiting_sequence(waiting)));
2729 }
2730 Ok(StreamMessage::Record(record, permit)) => {
2731 self.set_front_waiter(completion.worker, None);
2732 let buffered = BufferedRecord {
2733 worker: completion.worker,
2734 record,
2735 permit,
2736 };
2737 if !self.ordered {
2738 match self.consume(buffered) {
2739 Ok(true) => return Some(self.finish_batch()),
2740 Ok(false) => {}
2741 Err(error) => return Some(Err(error)),
2742 }
2743 continue;
2744 }
2745 let Some(sequence) = buffered.record.sequence else {
2746 self.run_context.cancel();
2747 if self.return_credit(buffered.worker).is_err() {
2748 self.close(true);
2749 return Some(Err(LoaderError::ChannelClosed {
2750 batch: self.next_batch,
2751 }));
2752 }
2753 return Some(Err(self.protocol_error(
2754 None,
2755 "ordered stream record did not provide a sequence identifier",
2756 )));
2757 };
2758 let sequence = sequence.get();
2759 if sequence < self.next_sequence {
2760 self.run_context.cancel();
2761 if self.return_credit(buffered.worker).is_err() {
2762 self.close(true);
2763 return Some(Err(LoaderError::ChannelClosed {
2764 batch: self.next_batch,
2765 }));
2766 }
2767 return Some(Err(self.protocol_error(
2768 Some(sequence),
2769 format!(
2770 "duplicate or past sequence {sequence}; next expected is {}",
2771 self.next_sequence
2772 ),
2773 )));
2774 }
2775 if self
2776 .completed
2777 .iter()
2778 .any(|(buffered_sequence, _)| *buffered_sequence == sequence)
2779 {
2780 self.run_context.cancel();
2781 if self.return_credit(buffered.worker).is_err() {
2782 self.close(true);
2783 return Some(Err(LoaderError::ChannelClosed {
2784 batch: self.next_batch,
2785 }));
2786 }
2787 return Some(Err(self.protocol_error(
2788 Some(sequence),
2789 format!("duplicate buffered sequence {sequence}"),
2790 )));
2791 }
2792 self.completed.push((sequence, buffered));
2793 }
2794 }
2795 }
2796 }
2797}
2798
2799impl<S, C, F, I, M, N> Drop for StreamLoaderIter<'_, S, C, F, I, M, N>
2800where
2801 S: WorkerSourceFactory,
2802 F: TransformFactory<S::Sample>,
2803 C: Collate<TransformOutput<S, F>>,
2804 I: WorkerInit,
2805 M: StreamMemoryPolicy,
2806{
2807 fn drop(&mut self) {
2808 self.run_context.cancel();
2809 let mut failed = false;
2810 while let Some((_, buffered)) = self.completed.pop() {
2811 failed |= self
2812 .pool
2813 .as_ref()
2814 .is_none_or(|pool| pool.pool().return_credit(buffered.worker).is_err());
2815 }
2816 if let Some(mut pool) = self.pool.take()
2817 && pool.is_persistent()
2818 && (failed || pool.pool_mut().quiesce(self.generation).is_err())
2819 {
2820 pool.pool_mut().poison();
2821 }
2822 }
2823}
2824
2825fn invalid_configuration(field: &'static str, reason: impl Into<String>) -> RustTorchError {
2826 RustTorchError::InvalidConfiguration {
2827 field,
2828 reason: reason.into(),
2829 }
2830}
2831
2832#[cfg(test)]
2833mod tests {
2834 use std::convert::Infallible;
2835
2836 use super::*;
2837
2838 type TestCompletion = StreamCompletion<
2839 usize,
2840 (),
2841 Infallible,
2842 Infallible,
2843 Infallible,
2844 Infallible,
2845 Infallible,
2846 Infallible,
2847 >;
2848 type TestResult = StreamResult<
2849 usize,
2850 (),
2851 Infallible,
2852 Infallible,
2853 Infallible,
2854 Infallible,
2855 Infallible,
2856 Infallible,
2857 >;
2858
2859 fn ready_completion(result: TestResult) -> TestCompletion {
2860 StreamCompletion {
2861 worker: 0,
2862 generation: 0,
2863 sequence: None,
2864 logical_id: None,
2865 holds_credit: false,
2866 result,
2867 }
2868 }
2869
2870 fn receive_after_expiry(completion: TestCompletion) -> TestCompletion {
2871 let (sender, receiver) = bounded(1);
2872 sender.send(completion).unwrap();
2873 let context = WorkerRunContext::new(0, 0, 0);
2874 context.deadline.arm(Duration::ZERO);
2875 let shutdown = CancellationToken::new();
2876 match receive_ready_first(&receiver, &context, &shutdown, Some(Duration::ZERO)) {
2877 ReadyFirst::Completion(completion) => completion,
2878 ReadyFirst::Timeout => panic!("ready completion lost to expired deadline"),
2879 ReadyFirst::Cancelled => panic!("ready completion lost to cancellation"),
2880 ReadyFirst::Closed => panic!("ready completion lost to channel close"),
2881 }
2882 }
2883
2884 #[test]
2885 fn queued_error_and_end_beat_an_expired_receive_deadline() {
2886 let error = receive_after_expiry(ready_completion(Err(StreamFailure::Panic)));
2887 assert!(matches!(error.result, Err(StreamFailure::Panic)));
2888
2889 let end = receive_after_expiry(ready_completion(Ok(StreamMessage::End)));
2890 assert!(matches!(end.result, Ok(StreamMessage::End)));
2891 }
2892}