rusttorch_data/lib.rs
1//! Prepare training and inference data with typed datasets and iterable batches.
2//!
3//! A [`Dataset`] supplies owned samples by index. A [sampler](Sampler) chooses
4//! their order, a [collator](Collate) combines them, and [`DataLoader`] produces
5//! batches on demand. Samples can be tensors, Rust records, or nested structures;
6//! dataset and pipeline failures are returned to the caller.
7//!
8//! The same API is available through [`rusttorch::data`](https://docs.rs/rusttorch/latest/rusttorch/data/).
9//! That module includes an end-to-end classifier training example; this crate
10//! focuses on preparing its inputs without depending on a model or optimizer.
11//!
12//! # Choose a loading pattern
13//!
14//! | Input or requirement | API |
15//! | --- | --- |
16//! | Features and labels already stored in tensors | [`TensorDataset`] |
17//! | Files or records addressable by index | Implement [`Dataset`] |
18//! | An existing fallible iterator | [`batches`], [`batches_with_collate`] |
19//! | Repeated epochs over indexed samples | [`DataLoader::builder`] |
20//! | Deterministic randomized sample order | [`RandomSampler`] |
21//! | Custom batch padding, packing, or conversion | [`FnCollate`] |
22//! | Explicitly partitioned worker streams | [`StreamDataLoaderBuilder`] |
23//! | Resume at a recorded iteration boundary | [`DataLoaderBuilder::resume_from`] |
24//!
25//! # Borrow once or own for repeated epochs
26//!
27//! [`DataLoader::new`] borrows a dataset and returns an iterator directly. It
28//! keeps samples in a `Vec`; use [`DataLoader::with_collate`] to change that.
29//! [`DataLoader::builder`] takes ownership, defaults to recursive tensor
30//! collation, and returns an [`OwnedDataLoader`] after `build()`. Call its
31//! `iter()` method for each pass and `set_epoch(epoch)` to advance seeded
32//! shuffling. Neither path collects the entire dataset before yielding a batch.
33//!
34//! # Batch paired tensors
35//!
36//! Store features and labels with the same first dimension. The default
37//! collator stacks corresponding sample fields: a dataset containing `[6, 3]`
38//! features and `[6]` labels yields `[2, 3]` features and `[2]` labels when the
39//! batch size is two.
40//!
41//! ```
42//! use rusttorch_core::{Device, Kind, Tensor};
43//! use rusttorch_data::{DataLoader, TensorDataset};
44//!
45//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
46//! let features = Tensor::f_zeros([6, 3], (Kind::Float, Device::Cpu))?;
47//! let labels = Tensor::f_from_slice(&[0_i64, 1, 0, 1, 0, 1])?;
48//! let dataset = TensorDataset::new(vec![features, labels])?;
49//! let mut loader = DataLoader::builder(dataset)
50//! .batch_size(2)
51//! .shuffle(42)?
52//! .build()?;
53//!
54//! for epoch in 0..2 {
55//! loader.set_epoch(epoch);
56//! for batch in loader.iter() {
57//! let batch = batch?;
58//! assert_eq!(batch[0].size(), [2, 3]);
59//! assert_eq!(batch[1].size(), [2]);
60//! // batch[0] is the model input; batch[1] contains its target classes.
61//! }
62//! }
63//! # Ok(())
64//! # }
65//! ```
66//!
67//! The loader owns the dataset; call `iter()` again for another pass. Use
68//! [`DataLoaderBuilder::drop_last`] when the model requires fixed-size batches.
69//! Tensor samples share storage with their dataset; avoid modifying views when
70//! the original values are needed for later epochs.
71//!
72//! A custom dataset returning `(Tensor, i64)` instead yields a typed
73//! `(Tensor, Tensor)` batch, so the training loop can use
74//! `let (features, labels) = batch?`. See [`Dataset`] for a working example.
75//!
76//! # Load your own records
77//!
78//! Implement [`Dataset`] to read a sample from a file, decode an image, or fetch
79//! a record from an indexed source. The associated error type preserves loading
80//! failures. Use [`VecCollate`] to keep records as Rust values:
81//!
82//! ```
83//! use std::convert::Infallible;
84//! use rusttorch_data::{DataLoader, Dataset, VecCollate};
85//!
86//! struct TextRows(Vec<String>);
87//!
88//! impl Dataset for TextRows {
89//! type Sample = String;
90//! type Error = Infallible;
91//!
92//! fn len(&self) -> usize { self.0.len() }
93//! fn get(&self, index: usize) -> Result<String, Infallible> {
94//! Ok(self.0[index].clone())
95//! }
96//! }
97//!
98//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
99//! let dataset = TextRows(vec!["first".into(), "second".into(), "third".into()]);
100//! let mut loader = DataLoader::builder(dataset)
101//! .batch_size(2)
102//! .collate(VecCollate)
103//! .build()?;
104//! let batches = loader.iter().collect::<Result<Vec<_>, _>>()?;
105//! assert_eq!(batches, [vec!["first", "second"], vec!["third"]]);
106//! # Ok(())
107//! # }
108//! ```
109//!
110//! # Customize batching and overlap preparation
111//!
112//! Start with serial loading, then use [`DataLoaderBuilder::workers`] to
113//! overlap sample preparation when your dataset satisfies the worker ownership
114//! requirements. A positive worker count shares the dataset through `Arc`;
115//! each worker fetches and transforms samples, and the calling thread collates
116//! them. [`DataLoaderBuilder::prefetch_factor`] bounds outstanding batches per
117//! worker. A dataset of file paths or ordinary Rust records can create new
118//! tensors during `get`; [`TensorDataset`] itself uses the serial path.
119//!
120//! [`DataLoaderBuilder`] demonstrates variable-length padding with [`FnCollate`]
121//! and inference without an extra batch dimension using `without_batching()`.
122//! [`StreamDataLoaderBuilder`] demonstrates lazy, independently owned shards
123//! when records do not have random-access indices. All shards are merged
124//! before batching, so `drop_last` drops at most one global tail.
125//!
126//! # Resume a training job
127//!
128//! Exact resumption requires replay-safe data and checkpointable pipeline
129//! components. [`LoaderState`] contains a complete map-loader save/restore
130//! example; [`ExactStreamDataLoader`] shows resumable stream cursors. Save at a
131//! successful batch boundary together with the corresponding model, optimizer
132//! and training counters. Loader checkpoints store the input position and
133//! component state, not model weights or decoded samples.
134
135#![deny(missing_docs)]
136
137use std::{convert::Infallible, marker::PhantomData};
138
139use rusttorch_core::{Result, RustTorchError};
140
141mod checkpoint;
142mod collate;
143mod dataset;
144mod error;
145mod limits;
146mod loader;
147mod memory;
148mod pin_memory;
149mod sampler;
150mod stream;
151mod transform;
152mod worker;
153mod worker_checkpoint;
154mod worker_context;
155
156pub use checkpoint::{
157 CheckpointActive, CheckpointBuildError, CheckpointDisabled, CheckpointFresh,
158 CheckpointIteration, CheckpointPinRequest, CheckpointPinStatus, CheckpointResume,
159 CheckpointSourceFactory, Checkpointable, CheckpointableSource, DatasetCheckpoint,
160 LOADER_STATE_SCHEMA_VERSION, LoaderConfiguration, LoaderState, ReplaySafeDataset, Stateless,
161 StatelessTransformFactory, StatelessWorker, StreamCheckpointBuildError,
162 StreamCheckpointConfiguration, StreamLaneState, StreamLoaderState, TransactionalCheckpoint,
163 TransactionalTransformFactory, TransactionalWorker, WorkerCheckpoint, WorkerCheckpointActive,
164 WorkerCheckpointIteration, WorkerLaneState, WorkerTransformLanes, WorkerTransformState,
165};
166pub use collate::{
167 Bytes, Collate, CollateError, DefaultCollate, DefaultCollator, DefaultConvert,
168 DefaultConverter, FnCollate, VecCollate,
169};
170pub use dataset::{
171 ConcatDataset, ReplaySafeMap, ReplaySafeTensorDataset, SplitLength, StackDataset, StackTuple,
172 Subset, TensorDataset, TransactionalMap, chain_datasets, random_split,
173};
174pub use error::{LoaderError, PipelineError};
175pub use limits::ResourceLimits;
176pub use loader::{
177 AutoBatch, BuilderDatasetMarker, CheckpointPlan, DataLoaderBuilder, ExplicitBatches,
178 LoaderIter, LoaderPlan, LoaderPlanConfiguration, NoBatch, OwnedDataLoader,
179 PlanCheckpointIdentity, SerialExecution, WorkerExecution, WorkerLoaderIter,
180};
181pub use memory::{MemoryDisabled, MemoryEnabled, MemoryFootprint};
182pub use pin_memory::{Auto, Explicit, PinDisabled, PinEnabled, PinMemory, PinMemoryStatus};
183pub use sampler::{
184 BatchSampler, BatchSamplerState, BatchSource, BatchSourceCheckpoint, DistributedConfiguration,
185 DistributedSampler, DistributedSamplerState, FnBatchSource, FnSampler, RandomReplacement,
186 RandomSampler, RandomSamplerState, Sampler, SamplerCheckpoint, SequentialSampler,
187 SequentialSamplerState, SubsetRandomSampler, SubsetRandomSamplerState, WeightedRandomSampler,
188 WeightedRandomSamplerState,
189};
190pub use stream::{
191 ExactStreamDataLoader, ExactStreamLoaderIter, LogicalSampleId, SequenceId, StreamDataLoader,
192 StreamDataLoaderBuilder, StreamLoaderIter, WorkerRecord, WorkerSourceFactory,
193};
194pub use transform::{
195 CloneTransformFactory, FnTransform, FnTransformFactory, IdentityTransform,
196 IdentityTransformFactory, TASK_RNG_DERIVATION_VERSION, TaskContext, Transform,
197 TransformFactory,
198};
199pub use worker_context::{
200 CancellationToken, Deadline, FnWorkerInit, LoaderCancelled, NoWorkerInit,
201 WORKER_SEED_DERIVATION_VERSION, WaitOutcome, WorkerContext, WorkerInfo, WorkerInit,
202 get_worker_info, with_worker_info,
203};
204
205/// A finite, indexable collection of samples.
206///
207/// Implementations return owned samples so loaders can move them into batches
208/// without cloning. Dataset-specific failures remain in [`Dataset::Error`].
209/// Keep `len()` stable during an iteration and make each sampled index valid
210/// for `get()`. Override [`Self::get_batch`] when one file read or database query
211/// can retrieve multiple rows more efficiently.
212///
213/// # Return typed feature/label pairs
214///
215/// The default collator stacks tensor features and converts integer labels into
216/// a tensor. Ordinary Rust storage can be shared by workers even though each
217/// fetched tensor is owned by a single sample.
218///
219/// ```
220/// use std::convert::Infallible;
221/// use rusttorch_core::Tensor;
222/// use rusttorch_data::{DataLoader, Dataset};
223///
224/// struct Rows(Vec<([f32; 2], i64)>);
225/// impl Dataset for Rows {
226/// type Sample = (Tensor, i64);
227/// type Error = Infallible;
228/// fn len(&self) -> usize { self.0.len() }
229/// fn get(&self, index: usize) -> Result<Self::Sample, Infallible> {
230/// let (values, label) = &self.0[index];
231/// Ok((Tensor::from_slice(values), *label))
232/// }
233/// }
234///
235/// fn main() -> Result<(), Box<dyn std::error::Error>> {
236/// let rows = Rows(vec![([1., 2.], 0), ([3., 4.], 1)]);
237/// let mut loader = DataLoader::builder(rows).batch_size(2).workers(2).build()?;
238/// let (features, labels) = loader.iter().next().unwrap()?;
239/// assert_eq!(features.size(), [2, 2]);
240/// assert_eq!(Vec::<i64>::try_from(&labels)?, [0, 1]);
241/// Ok(())
242/// }
243/// ```
244pub trait Dataset {
245 /// One owned item produced by the dataset.
246 type Sample;
247
248 /// An error returned while loading a sample.
249 type Error;
250
251 /// Returns the number of addressable samples.
252 fn len(&self) -> usize;
253
254 /// Loads the sample at `index`.
255 fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error>;
256
257 /// Loads samples for one index batch.
258 ///
259 /// The default calls [`Dataset::get`] once per index. Overrides must
260 /// return exactly one sample per supplied index and preserve index order.
261 fn get_batch(&self, indices: &[usize]) -> std::result::Result<Vec<Self::Sample>, Self::Error> {
262 indices.iter().map(|&index| self.get(index)).collect()
263 }
264
265 /// Loads one index batch with cooperative worker lifecycle context.
266 ///
267 /// The source-compatible default ignores `context` and delegates to
268 /// [`Dataset::get_batch`] exactly once. Context-aware datasets may
269 /// override this hook to honor cancellation and deadlines between bounded
270 /// operations. Rust cannot force-cancel an arbitrary blocking native call.
271 fn get_batch_with_context(
272 &self,
273 indices: &[usize],
274 context: &WorkerContext,
275 ) -> std::result::Result<Vec<Self::Sample>, Self::Error> {
276 let _ = context;
277 self.get_batch(indices)
278 }
279
280 /// Returns `true` when the dataset has no samples.
281 fn is_empty(&self) -> bool {
282 self.len() == 0
283 }
284
285 /// Iterates over every sample in index order.
286 ///
287 /// The iterator borrows the dataset and calls [`Dataset::get`] lazily.
288 fn samples(&self) -> DatasetSamples<'_, Self>
289 where
290 Self: Sized,
291 {
292 DatasetSamples {
293 dataset: self,
294 next_index: 0,
295 }
296 }
297}
298
299/// A borrowing, sequential iterator over a [`Dataset`].
300pub struct DatasetSamples<'a, D> {
301 dataset: &'a D,
302 next_index: usize,
303}
304
305impl<D> Iterator for DatasetSamples<'_, D>
306where
307 D: Dataset,
308{
309 type Item = std::result::Result<D::Sample, D::Error>;
310
311 fn next(&mut self) -> Option<Self::Item> {
312 if self.next_index >= self.dataset.len() {
313 return None;
314 }
315
316 let index = self.next_index;
317 self.next_index += 1;
318 Some(self.dataset.get(index))
319 }
320}
321
322type IdentityCollate<T, E> = fn(Vec<T>) -> std::result::Result<Vec<T>, E>;
323
324fn identity_batch<T, E>(samples: Vec<T>) -> std::result::Result<Vec<T>, E> {
325 Ok(samples)
326}
327
328struct DatasetBatchIterator<'a, D, S, C, B, E> {
329 dataset: &'a D,
330 sampler: S,
331 batch_size: usize,
332 drop_last: bool,
333 collate: C,
334 exhausted: bool,
335 output: PhantomData<fn() -> (B, E)>,
336}
337
338impl<D, S, C, B, E> Iterator for DatasetBatchIterator<'_, D, S, C, B, E>
339where
340 D: Dataset,
341 S: Iterator<Item = usize>,
342 C: FnMut(Vec<D::Sample>) -> std::result::Result<B, E>,
343 E: From<D::Error>,
344{
345 type Item = std::result::Result<B, E>;
346
347 fn next(&mut self) -> Option<Self::Item> {
348 if self.exhausted {
349 return None;
350 }
351
352 let mut indices = Vec::with_capacity(self.batch_size);
353 while indices.len() < self.batch_size {
354 match self.sampler.next() {
355 Some(index) => indices.push(index),
356 None => {
357 self.exhausted = true;
358 if indices.is_empty() {
359 return None;
360 }
361 break;
362 }
363 }
364 }
365
366 let samples = match self.dataset.get_batch(&indices) {
367 Ok(samples) => samples,
368 Err(error) => {
369 self.exhausted = true;
370 return Some(Err(E::from(error)));
371 }
372 };
373 assert_eq!(
374 samples.len(),
375 indices.len(),
376 "Dataset::get_batch returned {} samples for {} indices",
377 samples.len(),
378 indices.len()
379 );
380 if self.drop_last && indices.len() < self.batch_size {
381 return None;
382 }
383 let batch = (self.collate)(samples);
384 if batch.is_err() {
385 self.exhausted = true;
386 }
387 Some(batch)
388 }
389}
390
391struct BatchIterator<I, C, T, B, E> {
392 source: I,
393 batch_size: usize,
394 drop_last: bool,
395 collate: C,
396 exhausted: bool,
397 output: PhantomData<fn(T) -> (B, E)>,
398}
399
400impl<I, C, T, B, E> BatchIterator<I, C, T, B, E> {
401 fn new(source: I, batch_size: usize, drop_last: bool, collate: C) -> Result<Self> {
402 if batch_size == 0 {
403 return Err(RustTorchError::InvalidConfiguration {
404 field: "batch_size",
405 reason: "must be greater than zero".to_owned(),
406 });
407 }
408
409 Ok(Self {
410 source,
411 batch_size,
412 drop_last,
413 collate,
414 exhausted: false,
415 output: PhantomData,
416 })
417 }
418}
419
420impl<I, C, T, B, E> Iterator for BatchIterator<I, C, T, B, E>
421where
422 I: Iterator<Item = std::result::Result<T, E>>,
423 C: FnMut(Vec<T>) -> std::result::Result<B, E>,
424{
425 type Item = std::result::Result<B, E>;
426
427 fn next(&mut self) -> Option<Self::Item> {
428 if self.exhausted {
429 return None;
430 }
431
432 let first = match self.source.next() {
433 Some(Ok(sample)) => sample,
434 Some(Err(error)) => {
435 self.exhausted = true;
436 return Some(Err(error));
437 }
438 None => {
439 self.exhausted = true;
440 return None;
441 }
442 };
443
444 let mut samples = Vec::with_capacity(self.batch_size);
445 samples.push(first);
446 while samples.len() < self.batch_size {
447 match self.source.next() {
448 Some(Ok(sample)) => samples.push(sample),
449 Some(Err(error)) => {
450 self.exhausted = true;
451 return Some(Err(error));
452 }
453 None => {
454 self.exhausted = true;
455 if self.drop_last {
456 return None;
457 }
458 break;
459 }
460 }
461 }
462
463 let batch = (self.collate)(samples);
464 if batch.is_err() {
465 self.exhausted = true;
466 }
467 Some(batch)
468 }
469}
470
471/// Batches an ordinary fallible sample iterator into vectors.
472///
473/// Samples are moved into one pre-sized vector per batch. A short final batch
474/// is omitted when `drop_last` is `true`. Source errors are yielded once and
475/// then terminate the returned iterator.
476///
477/// # Errors
478///
479/// Returns [`RustTorchError::InvalidConfiguration`] when `batch_size` is zero.
480///
481/// # Examples
482///
483/// An ordinary fallible iterator is enough for streaming data:
484///
485/// ```
486/// use rusttorch_data::batches;
487///
488/// let records = ["10", "20", "30"].into_iter().map(str::parse::<i64>);
489/// let loader = batches(records, 2, false).expect("batch size is nonzero");
490/// let batches = loader
491/// .collect::<Result<Vec<_>, _>>()
492/// .expect("every record is an integer");
493///
494/// assert_eq!(batches, vec![vec![10, 20], vec![30]]);
495/// ```
496pub fn batches<I, T, E>(
497 source: I,
498 batch_size: usize,
499 drop_last: bool,
500) -> Result<impl Iterator<Item = std::result::Result<Vec<T>, E>>>
501where
502 I: Iterator<Item = std::result::Result<T, E>>,
503{
504 batches_with_collate(source, batch_size, drop_last, identity_batch::<T, E>)
505}
506
507/// Batches an ordinary fallible sample iterator with custom collation.
508///
509/// The closure receives ownership of exactly one vector of samples and may
510/// return any batch type. A short final batch is omitted when `drop_last` is
511/// `true`. Source and collation errors are yielded once and then terminate the
512/// returned iterator.
513///
514/// Use this for a parser or decoder that already yields records lazily. Here
515/// integer records become tensors without defining a dataset or worker factory.
516/// The source and closure use the same error type.
517///
518/// ```
519/// use rusttorch_core::Tensor;
520/// use rusttorch_data::batches_with_collate;
521/// fn main() -> Result<(), Box<dyn std::error::Error>> {
522/// let records = ["10", "20", "30"].into_iter().map(str::parse::<i64>);
523/// let mut loader = batches_with_collate(records, 2, false, |rows: Vec<i64>| {
524/// Ok::<_, std::num::ParseIntError>(Tensor::from_slice(&rows))
525/// })?;
526/// assert_eq!(Vec::<i64>::try_from(loader.next().unwrap()?)?, [10, 20]);
527/// assert_eq!(Vec::<i64>::try_from(loader.next().unwrap()?)?, [30]);
528/// assert!(loader.next().is_none());
529/// Ok(())
530/// }
531/// ```
532///
533/// # Errors
534///
535/// Returns [`RustTorchError::InvalidConfiguration`] when `batch_size` is zero.
536pub fn batches_with_collate<I, C, T, B, E>(
537 source: I,
538 batch_size: usize,
539 drop_last: bool,
540 collate: C,
541) -> Result<impl Iterator<Item = std::result::Result<B, E>>>
542where
543 I: Iterator<Item = std::result::Result<T, E>>,
544 C: FnMut(Vec<T>) -> std::result::Result<B, E>,
545{
546 BatchIterator::new(source, batch_size, drop_last, collate)
547}
548
549/// Load indexed samples through a borrowed iterator or a reusable owned loader.
550///
551/// Choose the constructor according to who should own the dataset:
552///
553/// | Constructor | Ownership and use | Default batch |
554/// | --- | --- | --- |
555/// | [`Self::new`] | Borrows `&dataset`; iterate the returned value directly | `Vec<Sample>` |
556/// | [`Self::with_collate`] | Same borrowed lifetime, with a collation closure | The closure's result |
557/// | [`Self::builder`] | Owns `dataset`; call `build()`, then `iter()` each epoch | [`DefaultCollator`] output |
558///
559/// The borrowed iterator fetches on the calling thread and yields errors once
560/// before ending. Samples are moved, so they need not implement `Clone`.
561/// For shuffling across epochs, worker prefetch, or checkpoint recovery, start
562/// with [`Self::builder`] and follow the [`DataLoaderBuilder`] examples.
563///
564/// # Examples
565///
566/// A seeded sampler and a collation closure can produce tensor batches:
567///
568/// ```
569/// use std::convert::Infallible;
570///
571/// use rusttorch_core::{Result, Tensor};
572/// use rusttorch_data::{DataLoader, Dataset, RandomSampler};
573///
574/// struct Rows(usize);
575///
576/// impl Dataset for Rows {
577/// type Sample = Tensor;
578/// type Error = Infallible;
579///
580/// fn len(&self) -> usize {
581/// self.0
582/// }
583///
584/// fn get(&self, index: usize) -> std::result::Result<Tensor, Infallible> {
585/// Ok(Tensor::from_slice(&[index as f32]))
586/// }
587/// }
588///
589/// fn main() -> Result<()> {
590/// let dataset = Rows(8);
591/// let sampler = RandomSampler::new(dataset.len(), 42)?;
592/// let loader = DataLoader::with_collate(
593/// &dataset,
594/// sampler,
595/// 4,
596/// false,
597/// |samples| Ok::<_, Infallible>(Tensor::stack(&samples, 0)),
598/// )?;
599///
600/// for batch in loader {
601/// let tensor = batch.expect("dataset and collation are infallible");
602/// assert_eq!(tensor.size(), [4, 1]);
603/// }
604/// Ok(())
605/// }
606/// ```
607pub struct DataLoader<
608 'a,
609 D = BuilderDatasetMarker,
610 S = std::iter::Empty<usize>,
611 C = (),
612 B = (),
613 E = Infallible,
614> where
615 D: Dataset,
616{
617 batches: DatasetBatchIterator<'a, D, S, C, B, E>,
618}
619
620impl DataLoader<'static, BuilderDatasetMarker, std::iter::Empty<usize>, (), (), Infallible> {
621 /// Starts an owned, re-iterable loader builder for `dataset`.
622 ///
623 /// Defaults are batch size one, sequential sampling, recursive collation,
624 /// zero workers, and keeping the final short batch. The builder's generic
625 /// parameters are inferred from the dataset and options; callers normally
626 /// do not write an explicit [`DataLoaderBuilder`] type.
627 ///
628 /// ```
629 /// use std::convert::Infallible;
630 /// use rusttorch_core::Result;
631 /// use rusttorch_data::{DataLoader, Dataset, VecCollate};
632 ///
633 /// struct Rows(Vec<i64>);
634 ///
635 /// impl Dataset for Rows {
636 /// type Sample = i64;
637 /// type Error = Infallible;
638 ///
639 /// fn len(&self) -> usize { self.0.len() }
640 /// fn get(&self, index: usize) -> std::result::Result<i64, Infallible> {
641 /// Ok(self.0[index])
642 /// }
643 /// }
644 ///
645 /// fn main() -> Result<()> {
646 /// let mut loader = DataLoader::builder(Rows(vec![1, 2, 3]))
647 /// .batch_size(2)
648 /// .collate(VecCollate)
649 /// .build()?;
650 /// let batches = loader.iter().collect::<std::result::Result<Vec<_>, _>>()
651 /// .expect("the dataset and VecCollate are infallible");
652 /// assert_eq!(batches, [vec![1, 2], vec![3]]);
653 /// Ok(())
654 /// }
655 /// ```
656 pub fn builder<D>(
657 dataset: D,
658 ) -> DataLoaderBuilder<D, AutoBatch<SequentialSampler>, DefaultCollator>
659 where
660 D: Dataset,
661 {
662 DataLoaderBuilder::new(dataset)
663 }
664}
665
666impl<'a, D, S> DataLoader<'a, D, S, IdentityCollate<D::Sample, D::Error>, Vec<D::Sample>, D::Error>
667where
668 D: Dataset,
669 S: Iterator<Item = usize>,
670{
671 /// Creates a loader whose batches are vectors of owned samples.
672 ///
673 /// `dataset` is borrowed, while `sampler` is consumed by the loader.
674 /// A short final batch is omitted when `drop_last` is `true`.
675 /// This constructor keeps each sample unchanged in a `Vec`; it does not
676 /// stack tensor rows. See [`Self::with_collate`] for tensor stacking or
677 /// [`Self::builder`] for default recursive collation.
678 ///
679 /// ```
680 /// use std::convert::Infallible;
681 /// use rusttorch_data::{DataLoader, Dataset};
682 /// struct Rows([i64; 3]);
683 /// impl Dataset for Rows {
684 /// type Sample = i64;
685 /// type Error = Infallible;
686 /// fn len(&self) -> usize { self.0.len() }
687 /// fn get(&self, i: usize) -> Result<i64, Infallible> { Ok(self.0[i]) }
688 /// }
689 /// fn main() -> Result<(), Box<dyn std::error::Error>> {
690 /// let dataset = Rows([5, 8, 13]);
691 /// let loader = DataLoader::new(&dataset, 0..dataset.len(), 2, false)?;
692 /// assert_eq!(loader.collect::<Result<Vec<_>, _>>()?, [vec![5, 8], vec![13]]);
693 /// assert_eq!(dataset.get(0)?, 5); // The dataset is still available.
694 /// Ok(())
695 /// }
696 /// ```
697 ///
698 /// # Errors
699 ///
700 /// Returns [`RustTorchError::InvalidConfiguration`] when `batch_size` is
701 /// zero.
702 pub fn new(dataset: &'a D, sampler: S, batch_size: usize, drop_last: bool) -> Result<Self> {
703 Self::with_collate(
704 dataset,
705 sampler,
706 batch_size,
707 drop_last,
708 identity_batch::<D::Sample, D::Error>,
709 )
710 }
711}
712
713impl<'a, D, S, C, B, E> DataLoader<'a, D, S, C, B, E>
714where
715 D: Dataset,
716 S: Iterator<Item = usize>,
717 C: FnMut(Vec<D::Sample>) -> std::result::Result<B, E>,
718 E: From<D::Error>,
719{
720 /// Creates a loader with fallible custom collation.
721 ///
722 /// The closure receives ownership of exactly one vector of samples and
723 /// may return any batch type. Dataset failures are converted into the
724 /// closure's error type through [`From`]. A short final batch is omitted
725 /// when `drop_last` is `true`.
726 ///
727 /// # Errors
728 ///
729 /// Returns [`RustTorchError::InvalidConfiguration`] when `batch_size` is
730 /// zero. Dataset and collation failures are returned by iteration.
731 pub fn with_collate(
732 dataset: &'a D,
733 sampler: S,
734 batch_size: usize,
735 drop_last: bool,
736 collate: C,
737 ) -> Result<Self> {
738 if batch_size == 0 {
739 return Err(RustTorchError::InvalidConfiguration {
740 field: "batch_size",
741 reason: "must be greater than zero".to_owned(),
742 });
743 }
744 Ok(Self {
745 batches: DatasetBatchIterator {
746 dataset,
747 sampler,
748 batch_size,
749 drop_last,
750 collate,
751 exhausted: false,
752 output: PhantomData,
753 },
754 })
755 }
756}
757
758impl<D, S, C, B, E> Iterator for DataLoader<'_, D, S, C, B, E>
759where
760 D: Dataset,
761 S: Iterator<Item = usize>,
762 C: FnMut(Vec<D::Sample>) -> std::result::Result<B, E>,
763 E: From<D::Error>,
764{
765 type Item = std::result::Result<B, E>;
766
767 fn next(&mut self) -> Option<Self::Item> {
768 self.batches.next()
769 }
770}