rusttorch_data/lib.rs
1//! Prepare training and inference data with typed datasets and iterable batches.
2//!
3//! A [`Dataset`] supplies owned samples by index. A [sampler](Sampler) chooses
4//! their order, a [collator](Collate) combines them, and [`DataLoader`] produces
5//! batches on demand. Samples can be tensors, Rust records, or nested structures;
6//! dataset and pipeline failures are returned to the caller.
7//!
8//! The same API is available through [`rusttorch::data`](https://docs.rs/rusttorch/latest/rusttorch/data/).
9//! That module includes an end-to-end classifier training example; this crate
10//! focuses on preparing its inputs without depending on a model or optimizer.
11//!
12//! # Choose a loading pattern
13//!
14//! | Input or requirement | API |
15//! | --- | --- |
16//! | Features and labels already stored in tensors | [`TensorDataset`] |
17//! | Files or records addressable by index | Implement [`Dataset`] |
18//! | An existing fallible iterator | [`batches`], [`batches_with_collate`] |
19//! | Repeated epochs over indexed samples | [`DataLoader::builder`] |
20//! | Deterministic randomized sample order | [`RandomSampler`] |
21//! | Custom batch padding, packing, or conversion | [`FnCollate`] |
22//! | Explicitly partitioned worker streams | [`StreamDataLoaderBuilder`] |
23//! | Resume at a recorded iteration boundary | [`DataLoaderBuilder::resume_from`] |
24//!
25//! # Borrow once or own for repeated epochs
26//!
27//! [`DataLoader::new`] borrows a dataset and returns an iterator directly. It
28//! keeps samples in a `Vec`; use [`DataLoader::with_collate`] to change that.
29//! [`DataLoader::builder`] takes ownership, defaults to recursive tensor
30//! collation, and returns an [`OwnedDataLoader`] after `build()`. Call its
31//! `iter()` method for each pass and `set_epoch(epoch)` to advance seeded
32//! shuffling. Neither path collects the entire dataset before yielding a batch.
33//!
34//! # Batch paired tensors
35//!
36//! Store features and labels with the same first dimension. The default
37//! collator stacks corresponding sample fields: a dataset containing `[6, 3]`
38//! features and `[6]` labels yields `[2, 3]` features and `[2]` labels when the
39//! batch size is two.
40//!
41//! ```
42//! use rusttorch_core::{Device, Kind, Tensor};
43//! use rusttorch_data::{DataLoader, TensorDataset};
44//!
45//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
46//! let features = Tensor::f_zeros([6, 3], (Kind::Float, Device::Cpu))?;
47//! let labels = Tensor::f_from_slice(&[0_i64, 1, 0, 1, 0, 1])?;
48//! let dataset = TensorDataset::new(vec![features, labels])?;
49//! let mut loader = DataLoader::builder(dataset)
50//! .batch_size(2)
51//! .shuffle(42)?
52//! .build()?;
53//!
54//! for epoch in 0..2 {
55//! loader.set_epoch(epoch);
56//! for batch in loader.iter() {
57//! let batch = batch?;
58//! assert_eq!(batch[0].size(), [2, 3]);
59//! assert_eq!(batch[1].size(), [2]);
60//! // batch[0] is the model input; batch[1] contains its target classes.
61//! }
62//! }
63//! # Ok(())
64//! # }
65//! ```
66//!
67//! The loader owns the dataset; call `iter()` again for another pass. Use
68//! [`DataLoaderBuilder::drop_last`] when the model requires fixed-size batches.
69//! Tensor samples share storage with their dataset; avoid modifying views when
70//! the original values are needed for later epochs.
71//!
72//! A custom dataset returning `(Tensor, i64)` instead yields a typed
73//! `(Tensor, Tensor)` batch, so the training loop can use
74//! `let (features, labels) = batch?`. See [`Dataset`] for a working example.
75//!
76//! # Load your own records
77//!
78//! Implement [`Dataset`] to read a sample from a file, decode an image, or fetch
79//! a record from an indexed source. The associated error type preserves loading
80//! failures. Use [`VecCollate`] to keep records as Rust values:
81//!
82//! ```
83//! use std::convert::Infallible;
84//! use rusttorch_data::{DataLoader, Dataset, VecCollate};
85//!
86//! struct TextRows(Vec<String>);
87//!
88//! impl Dataset for TextRows {
89//! type Sample = String;
90//! type Error = Infallible;
91//!
92//! fn len(&self) -> usize { self.0.len() }
93//! fn get(&self, index: usize) -> Result<String, Infallible> {
94//! Ok(self.0[index].clone())
95//! }
96//! }
97//!
98//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
99//! let dataset = TextRows(vec!["first".into(), "second".into(), "third".into()]);
100//! let mut loader = DataLoader::builder(dataset)
101//! .batch_size(2)
102//! .collate(VecCollate)
103//! .build()?;
104//! let batches = loader.iter().collect::<Result<Vec<_>, _>>()?;
105//! assert_eq!(batches, [vec!["first", "second"], vec!["third"]]);
106//! # Ok(())
107//! # }
108//! ```
109//!
110//! # Customize batching and overlap preparation
111//!
112//! Start with serial loading, then use [`DataLoaderBuilder::workers`] to
113//! overlap sample preparation when your dataset satisfies the worker ownership
114//! requirements. A positive worker count shares the dataset through `Arc`;
115//! each worker fetches and transforms samples, and the calling thread collates
116//! them. [`DataLoaderBuilder::prefetch_factor`] bounds outstanding batches per
117//! worker. A dataset of file paths or ordinary Rust records can create new
118//! tensors during `get`; [`TensorDataset`] itself uses the serial path.
119//!
120//! [`DataLoaderBuilder`] demonstrates variable-length padding with [`FnCollate`]
121//! and inference without an extra batch dimension using `without_batching()`.
122//! [`StreamDataLoaderBuilder`] demonstrates lazy, independently owned shards
123//! when records do not have random-access indices. All shards are merged
124//! before batching, so `drop_last` drops at most one global tail.
125//!
126//! # Resume a training job
127//!
128//! Exact resumption requires replay-safe data and checkpointable pipeline
129//! components. [`LoaderState`] contains a complete map-loader save/restore
130//! example; [`ExactStreamDataLoader`] shows resumable stream cursors. Save at a
131//! successful batch boundary together with the corresponding model, optimizer
132//! and training counters. Loader checkpoints store the input position and
133//! component state, not model weights or decoded samples.
134
135#![deny(missing_docs)]
136
137use std::{convert::Infallible, marker::PhantomData};
138
139use rusttorch_core::{Result, RustTorchError};
140
141mod checkpoint;
142mod collate;
143mod dataset;
144mod error;
145mod loader;
146mod memory;
147mod pin_memory;
148mod sampler;
149mod stream;
150mod transform;
151mod worker;
152mod worker_checkpoint;
153mod worker_context;
154
155pub use checkpoint::{
156 CheckpointActive, CheckpointBuildError, CheckpointDisabled, CheckpointFresh,
157 CheckpointIteration, CheckpointPinRequest, CheckpointPinStatus, CheckpointResume,
158 CheckpointSourceFactory, Checkpointable, CheckpointableSource, DatasetCheckpoint,
159 LOADER_STATE_SCHEMA_VERSION, LoaderConfiguration, LoaderState, ReplaySafeDataset, Stateless,
160 StatelessTransformFactory, StatelessWorker, StreamCheckpointBuildError,
161 StreamCheckpointConfiguration, StreamLaneState, StreamLoaderState, TransactionalCheckpoint,
162 TransactionalTransformFactory, TransactionalWorker, WorkerCheckpoint, WorkerCheckpointActive,
163 WorkerCheckpointIteration, WorkerLaneState, WorkerTransformLanes, WorkerTransformState,
164};
165pub use collate::{
166 Bytes, Collate, CollateError, DefaultCollate, DefaultCollator, DefaultConvert,
167 DefaultConverter, FnCollate, VecCollate,
168};
169pub use dataset::{
170 ConcatDataset, ReplaySafeMap, ReplaySafeTensorDataset, SplitLength, StackDataset, StackTuple,
171 Subset, TensorDataset, TransactionalMap, chain_datasets, random_split,
172};
173pub use error::{LoaderError, PipelineError};
174pub use loader::{
175 AutoBatch, BuilderDatasetMarker, CheckpointPlan, DataLoaderBuilder, ExplicitBatches,
176 LoaderIter, LoaderPlan, LoaderPlanConfiguration, NoBatch, OwnedDataLoader,
177 PlanCheckpointIdentity, SerialExecution, WorkerExecution, WorkerLoaderIter,
178};
179pub use memory::{MemoryDisabled, MemoryEnabled, MemoryFootprint};
180pub use pin_memory::{Auto, Explicit, PinDisabled, PinEnabled, PinMemory, PinMemoryStatus};
181pub use sampler::{
182 BatchSampler, BatchSamplerState, BatchSource, BatchSourceCheckpoint, DistributedConfiguration,
183 DistributedSampler, DistributedSamplerState, FnBatchSource, FnSampler, RandomReplacement,
184 RandomSampler, RandomSamplerState, Sampler, SamplerCheckpoint, SequentialSampler,
185 SequentialSamplerState, SubsetRandomSampler, SubsetRandomSamplerState, WeightedRandomSampler,
186 WeightedRandomSamplerState,
187};
188pub use stream::{
189 ExactStreamDataLoader, ExactStreamLoaderIter, LogicalSampleId, SequenceId, StreamDataLoader,
190 StreamDataLoaderBuilder, StreamLoaderIter, WorkerRecord, WorkerSourceFactory,
191};
192pub use transform::{
193 CloneTransformFactory, FnTransform, FnTransformFactory, IdentityTransform,
194 IdentityTransformFactory, TASK_RNG_DERIVATION_VERSION, TaskContext, Transform,
195 TransformFactory,
196};
197pub use worker_context::{
198 CancellationToken, Deadline, FnWorkerInit, LoaderCancelled, NoWorkerInit,
199 WORKER_SEED_DERIVATION_VERSION, WaitOutcome, WorkerContext, WorkerInfo, WorkerInit,
200 get_worker_info, with_worker_info,
201};
202
203/// A finite, indexable collection of samples.
204///
205/// Implementations return owned samples so loaders can move them into batches
206/// without cloning. Dataset-specific failures remain in [`Dataset::Error`].
207/// Keep `len()` stable during an iteration and make each sampled index valid
208/// for `get()`. Override [`Self::get_batch`] when one file read or database query
209/// can retrieve multiple rows more efficiently.
210///
211/// # Return typed feature/label pairs
212///
213/// The default collator stacks tensor features and converts integer labels into
214/// a tensor. Ordinary Rust storage can be shared by workers even though each
215/// fetched tensor is owned by a single sample.
216///
217/// ```
218/// use std::convert::Infallible;
219/// use rusttorch_core::Tensor;
220/// use rusttorch_data::{DataLoader, Dataset};
221///
222/// struct Rows(Vec<([f32; 2], i64)>);
223/// impl Dataset for Rows {
224/// type Sample = (Tensor, i64);
225/// type Error = Infallible;
226/// fn len(&self) -> usize { self.0.len() }
227/// fn get(&self, index: usize) -> Result<Self::Sample, Infallible> {
228/// let (values, label) = &self.0[index];
229/// Ok((Tensor::from_slice(values), *label))
230/// }
231/// }
232///
233/// fn main() -> Result<(), Box<dyn std::error::Error>> {
234/// let rows = Rows(vec![([1., 2.], 0), ([3., 4.], 1)]);
235/// let mut loader = DataLoader::builder(rows).batch_size(2).workers(2).build()?;
236/// let (features, labels) = loader.iter().next().unwrap()?;
237/// assert_eq!(features.size(), [2, 2]);
238/// assert_eq!(Vec::<i64>::try_from(&labels)?, [0, 1]);
239/// Ok(())
240/// }
241/// ```
242pub trait Dataset {
243 /// One owned item produced by the dataset.
244 type Sample;
245
246 /// An error returned while loading a sample.
247 type Error;
248
249 /// Returns the number of addressable samples.
250 fn len(&self) -> usize;
251
252 /// Loads the sample at `index`.
253 fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error>;
254
255 /// Loads samples for one index batch.
256 ///
257 /// The default calls [`Dataset::get`] once per index. Overrides must
258 /// return exactly one sample per supplied index and preserve index order.
259 fn get_batch(&self, indices: &[usize]) -> std::result::Result<Vec<Self::Sample>, Self::Error> {
260 indices.iter().map(|&index| self.get(index)).collect()
261 }
262
263 /// Loads one index batch with cooperative worker lifecycle context.
264 ///
265 /// The source-compatible default ignores `context` and delegates to
266 /// [`Dataset::get_batch`] exactly once. Context-aware datasets may
267 /// override this hook to honor cancellation and deadlines between bounded
268 /// operations. Rust cannot force-cancel an arbitrary blocking native call.
269 fn get_batch_with_context(
270 &self,
271 indices: &[usize],
272 context: &WorkerContext,
273 ) -> std::result::Result<Vec<Self::Sample>, Self::Error> {
274 let _ = context;
275 self.get_batch(indices)
276 }
277
278 /// Returns `true` when the dataset has no samples.
279 fn is_empty(&self) -> bool {
280 self.len() == 0
281 }
282
283 /// Iterates over every sample in index order.
284 ///
285 /// The iterator borrows the dataset and calls [`Dataset::get`] lazily.
286 fn samples(&self) -> DatasetSamples<'_, Self>
287 where
288 Self: Sized,
289 {
290 DatasetSamples {
291 dataset: self,
292 next_index: 0,
293 }
294 }
295}
296
297/// A borrowing, sequential iterator over a [`Dataset`].
298pub struct DatasetSamples<'a, D> {
299 dataset: &'a D,
300 next_index: usize,
301}
302
303impl<D> Iterator for DatasetSamples<'_, D>
304where
305 D: Dataset,
306{
307 type Item = std::result::Result<D::Sample, D::Error>;
308
309 fn next(&mut self) -> Option<Self::Item> {
310 if self.next_index >= self.dataset.len() {
311 return None;
312 }
313
314 let index = self.next_index;
315 self.next_index += 1;
316 Some(self.dataset.get(index))
317 }
318}
319
320type IdentityCollate<T, E> = fn(Vec<T>) -> std::result::Result<Vec<T>, E>;
321
322fn identity_batch<T, E>(samples: Vec<T>) -> std::result::Result<Vec<T>, E> {
323 Ok(samples)
324}
325
326struct DatasetBatchIterator<'a, D, S, C, B, E> {
327 dataset: &'a D,
328 sampler: S,
329 batch_size: usize,
330 drop_last: bool,
331 collate: C,
332 exhausted: bool,
333 output: PhantomData<fn() -> (B, E)>,
334}
335
336impl<D, S, C, B, E> Iterator for DatasetBatchIterator<'_, D, S, C, B, E>
337where
338 D: Dataset,
339 S: Iterator<Item = usize>,
340 C: FnMut(Vec<D::Sample>) -> std::result::Result<B, E>,
341 E: From<D::Error>,
342{
343 type Item = std::result::Result<B, E>;
344
345 fn next(&mut self) -> Option<Self::Item> {
346 if self.exhausted {
347 return None;
348 }
349
350 let mut indices = Vec::with_capacity(self.batch_size);
351 while indices.len() < self.batch_size {
352 match self.sampler.next() {
353 Some(index) => indices.push(index),
354 None => {
355 self.exhausted = true;
356 if indices.is_empty() {
357 return None;
358 }
359 break;
360 }
361 }
362 }
363
364 let samples = match self.dataset.get_batch(&indices) {
365 Ok(samples) => samples,
366 Err(error) => {
367 self.exhausted = true;
368 return Some(Err(E::from(error)));
369 }
370 };
371 assert_eq!(
372 samples.len(),
373 indices.len(),
374 "Dataset::get_batch returned {} samples for {} indices",
375 samples.len(),
376 indices.len()
377 );
378 if self.drop_last && indices.len() < self.batch_size {
379 return None;
380 }
381 let batch = (self.collate)(samples);
382 if batch.is_err() {
383 self.exhausted = true;
384 }
385 Some(batch)
386 }
387}
388
389struct BatchIterator<I, C, T, B, E> {
390 source: I,
391 batch_size: usize,
392 drop_last: bool,
393 collate: C,
394 exhausted: bool,
395 output: PhantomData<fn(T) -> (B, E)>,
396}
397
398impl<I, C, T, B, E> BatchIterator<I, C, T, B, E> {
399 fn new(source: I, batch_size: usize, drop_last: bool, collate: C) -> Result<Self> {
400 if batch_size == 0 {
401 return Err(RustTorchError::InvalidConfiguration {
402 field: "batch_size",
403 reason: "must be greater than zero".to_owned(),
404 });
405 }
406
407 Ok(Self {
408 source,
409 batch_size,
410 drop_last,
411 collate,
412 exhausted: false,
413 output: PhantomData,
414 })
415 }
416}
417
418impl<I, C, T, B, E> Iterator for BatchIterator<I, C, T, B, E>
419where
420 I: Iterator<Item = std::result::Result<T, E>>,
421 C: FnMut(Vec<T>) -> std::result::Result<B, E>,
422{
423 type Item = std::result::Result<B, E>;
424
425 fn next(&mut self) -> Option<Self::Item> {
426 if self.exhausted {
427 return None;
428 }
429
430 let first = match self.source.next() {
431 Some(Ok(sample)) => sample,
432 Some(Err(error)) => {
433 self.exhausted = true;
434 return Some(Err(error));
435 }
436 None => {
437 self.exhausted = true;
438 return None;
439 }
440 };
441
442 let mut samples = Vec::with_capacity(self.batch_size);
443 samples.push(first);
444 while samples.len() < self.batch_size {
445 match self.source.next() {
446 Some(Ok(sample)) => samples.push(sample),
447 Some(Err(error)) => {
448 self.exhausted = true;
449 return Some(Err(error));
450 }
451 None => {
452 self.exhausted = true;
453 if self.drop_last {
454 return None;
455 }
456 break;
457 }
458 }
459 }
460
461 let batch = (self.collate)(samples);
462 if batch.is_err() {
463 self.exhausted = true;
464 }
465 Some(batch)
466 }
467}
468
469/// Batches an ordinary fallible sample iterator into vectors.
470///
471/// Samples are moved into one pre-sized vector per batch. A short final batch
472/// is omitted when `drop_last` is `true`. Source errors are yielded once and
473/// then terminate the returned iterator.
474///
475/// # Errors
476///
477/// Returns [`RustTorchError::InvalidConfiguration`] when `batch_size` is zero.
478///
479/// # Examples
480///
481/// An ordinary fallible iterator is enough for streaming data:
482///
483/// ```
484/// use rusttorch_data::batches;
485///
486/// let records = ["10", "20", "30"].into_iter().map(str::parse::<i64>);
487/// let loader = batches(records, 2, false).expect("batch size is nonzero");
488/// let batches = loader
489/// .collect::<Result<Vec<_>, _>>()
490/// .expect("every record is an integer");
491///
492/// assert_eq!(batches, vec![vec![10, 20], vec![30]]);
493/// ```
494pub fn batches<I, T, E>(
495 source: I,
496 batch_size: usize,
497 drop_last: bool,
498) -> Result<impl Iterator<Item = std::result::Result<Vec<T>, E>>>
499where
500 I: Iterator<Item = std::result::Result<T, E>>,
501{
502 batches_with_collate(source, batch_size, drop_last, identity_batch::<T, E>)
503}
504
505/// Batches an ordinary fallible sample iterator with custom collation.
506///
507/// The closure receives ownership of exactly one vector of samples and may
508/// return any batch type. A short final batch is omitted when `drop_last` is
509/// `true`. Source and collation errors are yielded once and then terminate the
510/// returned iterator.
511///
512/// Use this for a parser or decoder that already yields records lazily. Here
513/// integer records become tensors without defining a dataset or worker factory.
514/// The source and closure use the same error type.
515///
516/// ```
517/// use rusttorch_core::Tensor;
518/// use rusttorch_data::batches_with_collate;
519/// fn main() -> Result<(), Box<dyn std::error::Error>> {
520/// let records = ["10", "20", "30"].into_iter().map(str::parse::<i64>);
521/// let mut loader = batches_with_collate(records, 2, false, |rows: Vec<i64>| {
522/// Ok::<_, std::num::ParseIntError>(Tensor::from_slice(&rows))
523/// })?;
524/// assert_eq!(Vec::<i64>::try_from(loader.next().unwrap()?)?, [10, 20]);
525/// assert_eq!(Vec::<i64>::try_from(loader.next().unwrap()?)?, [30]);
526/// assert!(loader.next().is_none());
527/// Ok(())
528/// }
529/// ```
530///
531/// # Errors
532///
533/// Returns [`RustTorchError::InvalidConfiguration`] when `batch_size` is zero.
534pub fn batches_with_collate<I, C, T, B, E>(
535 source: I,
536 batch_size: usize,
537 drop_last: bool,
538 collate: C,
539) -> Result<impl Iterator<Item = std::result::Result<B, E>>>
540where
541 I: Iterator<Item = std::result::Result<T, E>>,
542 C: FnMut(Vec<T>) -> std::result::Result<B, E>,
543{
544 BatchIterator::new(source, batch_size, drop_last, collate)
545}
546
547/// Load indexed samples through a borrowed iterator or a reusable owned loader.
548///
549/// Choose the constructor according to who should own the dataset:
550///
551/// | Constructor | Ownership and use | Default batch |
552/// | --- | --- | --- |
553/// | [`Self::new`] | Borrows `&dataset`; iterate the returned value directly | `Vec<Sample>` |
554/// | [`Self::with_collate`] | Same borrowed lifetime, with a collation closure | The closure's result |
555/// | [`Self::builder`] | Owns `dataset`; call `build()`, then `iter()` each epoch | [`DefaultCollator`] output |
556///
557/// The borrowed iterator fetches on the calling thread and yields errors once
558/// before ending. Samples are moved, so they need not implement `Clone`.
559/// For shuffling across epochs, worker prefetch, or checkpoint recovery, start
560/// with [`Self::builder`] and follow the [`DataLoaderBuilder`] examples.
561///
562/// # Examples
563///
564/// A seeded sampler and a collation closure can produce tensor batches:
565///
566/// ```
567/// use std::convert::Infallible;
568///
569/// use rusttorch_core::{Result, Tensor};
570/// use rusttorch_data::{DataLoader, Dataset, RandomSampler};
571///
572/// struct Rows(usize);
573///
574/// impl Dataset for Rows {
575/// type Sample = Tensor;
576/// type Error = Infallible;
577///
578/// fn len(&self) -> usize {
579/// self.0
580/// }
581///
582/// fn get(&self, index: usize) -> std::result::Result<Tensor, Infallible> {
583/// Ok(Tensor::from_slice(&[index as f32]))
584/// }
585/// }
586///
587/// fn main() -> Result<()> {
588/// let dataset = Rows(8);
589/// let sampler = RandomSampler::new(dataset.len(), 42)?;
590/// let loader = DataLoader::with_collate(
591/// &dataset,
592/// sampler,
593/// 4,
594/// false,
595/// |samples| Ok::<_, Infallible>(Tensor::stack(&samples, 0)),
596/// )?;
597///
598/// for batch in loader {
599/// let tensor = batch.expect("dataset and collation are infallible");
600/// assert_eq!(tensor.size(), [4, 1]);
601/// }
602/// Ok(())
603/// }
604/// ```
605pub struct DataLoader<
606 'a,
607 D = BuilderDatasetMarker,
608 S = std::iter::Empty<usize>,
609 C = (),
610 B = (),
611 E = Infallible,
612> where
613 D: Dataset,
614{
615 batches: DatasetBatchIterator<'a, D, S, C, B, E>,
616}
617
618impl DataLoader<'static, BuilderDatasetMarker, std::iter::Empty<usize>, (), (), Infallible> {
619 /// Starts an owned, re-iterable loader builder for `dataset`.
620 ///
621 /// Defaults are batch size one, sequential sampling, recursive collation,
622 /// zero workers, and keeping the final short batch. The builder's generic
623 /// parameters are inferred from the dataset and options; callers normally
624 /// do not write an explicit [`DataLoaderBuilder`] type.
625 ///
626 /// ```
627 /// use std::convert::Infallible;
628 /// use rusttorch_core::Result;
629 /// use rusttorch_data::{DataLoader, Dataset, VecCollate};
630 ///
631 /// struct Rows(Vec<i64>);
632 ///
633 /// impl Dataset for Rows {
634 /// type Sample = i64;
635 /// type Error = Infallible;
636 ///
637 /// fn len(&self) -> usize { self.0.len() }
638 /// fn get(&self, index: usize) -> std::result::Result<i64, Infallible> {
639 /// Ok(self.0[index])
640 /// }
641 /// }
642 ///
643 /// fn main() -> Result<()> {
644 /// let mut loader = DataLoader::builder(Rows(vec![1, 2, 3]))
645 /// .batch_size(2)
646 /// .collate(VecCollate)
647 /// .build()?;
648 /// let batches = loader.iter().collect::<std::result::Result<Vec<_>, _>>()
649 /// .expect("the dataset and VecCollate are infallible");
650 /// assert_eq!(batches, [vec![1, 2], vec![3]]);
651 /// Ok(())
652 /// }
653 /// ```
654 pub fn builder<D>(
655 dataset: D,
656 ) -> DataLoaderBuilder<D, AutoBatch<SequentialSampler>, DefaultCollator>
657 where
658 D: Dataset,
659 {
660 DataLoaderBuilder::new(dataset)
661 }
662}
663
664impl<'a, D, S> DataLoader<'a, D, S, IdentityCollate<D::Sample, D::Error>, Vec<D::Sample>, D::Error>
665where
666 D: Dataset,
667 S: Iterator<Item = usize>,
668{
669 /// Creates a loader whose batches are vectors of owned samples.
670 ///
671 /// `dataset` is borrowed, while `sampler` is consumed by the loader.
672 /// A short final batch is omitted when `drop_last` is `true`.
673 /// This constructor keeps each sample unchanged in a `Vec`; it does not
674 /// stack tensor rows. See [`Self::with_collate`] for tensor stacking or
675 /// [`Self::builder`] for default recursive collation.
676 ///
677 /// ```
678 /// use std::convert::Infallible;
679 /// use rusttorch_data::{DataLoader, Dataset};
680 /// struct Rows([i64; 3]);
681 /// impl Dataset for Rows {
682 /// type Sample = i64;
683 /// type Error = Infallible;
684 /// fn len(&self) -> usize { self.0.len() }
685 /// fn get(&self, i: usize) -> Result<i64, Infallible> { Ok(self.0[i]) }
686 /// }
687 /// fn main() -> Result<(), Box<dyn std::error::Error>> {
688 /// let dataset = Rows([5, 8, 13]);
689 /// let loader = DataLoader::new(&dataset, 0..dataset.len(), 2, false)?;
690 /// assert_eq!(loader.collect::<Result<Vec<_>, _>>()?, [vec![5, 8], vec![13]]);
691 /// assert_eq!(dataset.get(0)?, 5); // The dataset is still available.
692 /// Ok(())
693 /// }
694 /// ```
695 ///
696 /// # Errors
697 ///
698 /// Returns [`RustTorchError::InvalidConfiguration`] when `batch_size` is
699 /// zero.
700 pub fn new(dataset: &'a D, sampler: S, batch_size: usize, drop_last: bool) -> Result<Self> {
701 Self::with_collate(
702 dataset,
703 sampler,
704 batch_size,
705 drop_last,
706 identity_batch::<D::Sample, D::Error>,
707 )
708 }
709}
710
711impl<'a, D, S, C, B, E> DataLoader<'a, D, S, C, B, E>
712where
713 D: Dataset,
714 S: Iterator<Item = usize>,
715 C: FnMut(Vec<D::Sample>) -> std::result::Result<B, E>,
716 E: From<D::Error>,
717{
718 /// Creates a loader with fallible custom collation.
719 ///
720 /// The closure receives ownership of exactly one vector of samples and
721 /// may return any batch type. Dataset failures are converted into the
722 /// closure's error type through [`From`]. A short final batch is omitted
723 /// when `drop_last` is `true`.
724 ///
725 /// # Errors
726 ///
727 /// Returns [`RustTorchError::InvalidConfiguration`] when `batch_size` is
728 /// zero. Dataset and collation failures are returned by iteration.
729 pub fn with_collate(
730 dataset: &'a D,
731 sampler: S,
732 batch_size: usize,
733 drop_last: bool,
734 collate: C,
735 ) -> Result<Self> {
736 if batch_size == 0 {
737 return Err(RustTorchError::InvalidConfiguration {
738 field: "batch_size",
739 reason: "must be greater than zero".to_owned(),
740 });
741 }
742 Ok(Self {
743 batches: DatasetBatchIterator {
744 dataset,
745 sampler,
746 batch_size,
747 drop_last,
748 collate,
749 exhausted: false,
750 output: PhantomData,
751 },
752 })
753 }
754}
755
756impl<D, S, C, B, E> Iterator for DataLoader<'_, D, S, C, B, E>
757where
758 D: Dataset,
759 S: Iterator<Item = usize>,
760 C: FnMut(Vec<D::Sample>) -> std::result::Result<B, E>,
761 E: From<D::Error>,
762{
763 type Item = std::result::Result<B, E>;
764
765 fn next(&mut self) -> Option<Self::Item> {
766 self.batches.next()
767 }
768}