Skip to main content

rusttorch_data/
dataset.rs

1use std::sync::Arc;
2
3use rand::{SeedableRng, seq::SliceRandom};
4use rand_chacha::ChaCha12Rng;
5use rusttorch_core::{Result, RustTorchError, Tensor};
6
7use crate::{Checkpointable, Dataset, DatasetCheckpoint, ReplaySafeDataset, WorkerContext};
8
9/// A map-style dataset backed by tensors with a shared first dimension.
10///
11/// Indexed samples are ordinary LibTorch row views and therefore share
12/// storage with the tensors supplied to [`TensorDataset::new`].
13pub struct TensorDataset {
14    tensors: Vec<Tensor>,
15    len: usize,
16}
17
18impl TensorDataset {
19    /// Creates a tensor dataset whose tensors have the same non-scalar first dimension.
20    ///
21    /// # Errors
22    ///
23    /// Returns [`RustTorchError::InvalidConfiguration`] for no tensors and
24    /// [`RustTorchError::InvalidDimensions`] for a scalar tensor or mismatched
25    /// first dimensions.
26    pub fn new(tensors: Vec<Tensor>) -> Result<Self> {
27        let first = tensors
28            .first()
29            .ok_or_else(|| RustTorchError::InvalidConfiguration {
30                field: "tensors",
31                reason: "must not be empty".to_owned(),
32            })?;
33        let first_size = first.size();
34        let len = first_size
35            .first()
36            .copied()
37            .ok_or_else(|| RustTorchError::InvalidDimensions {
38                context: "TensorDataset tensor 0".to_owned(),
39                expected: "at least one dimension".to_owned(),
40                actual: format!("{first_size:?}"),
41            })?;
42
43        for (index, tensor) in tensors.iter().enumerate().skip(1) {
44            let size = tensor.size();
45            if size.first().copied() != Some(len) {
46                return Err(RustTorchError::InvalidDimensions {
47                    context: format!("TensorDataset tensor {index}"),
48                    expected: format!("first dimension {len}"),
49                    actual: format!("{size:?}"),
50                });
51            }
52        }
53
54        Ok(Self {
55            tensors,
56            len: usize::try_from(len).map_err(|_| RustTorchError::InvalidDimensions {
57                context: "TensorDataset tensor 0".to_owned(),
58                expected: "a nonnegative first dimension representable as usize".to_owned(),
59                actual: len.to_string(),
60            })?,
61        })
62    }
63
64    /// Deep-copies the backing tensors for exact replay-safe loading.
65    ///
66    /// The returned dataset also deep-copies every fetched row, so callers
67    /// cannot mutate its private backing storage through a sample view.
68    ///
69    /// # Errors
70    ///
71    /// Returns the preserved LibTorch backend error when allocation or copy
72    /// fails.
73    pub fn into_replay_safe(self) -> Result<ReplaySafeTensorDataset> {
74        let tensors = self
75            .tensors
76            .iter()
77            .map(deep_copy_tensor)
78            .collect::<Result<Vec<_>>>()?;
79        Ok(ReplaySafeTensorDataset {
80            tensors,
81            len: self.len,
82        })
83    }
84}
85
86impl Dataset for TensorDataset {
87    type Sample = Vec<Tensor>;
88    type Error = RustTorchError;
89
90    fn len(&self) -> usize {
91        self.len
92    }
93
94    fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error> {
95        let index = i64::try_from(index).map_err(|_| RustTorchError::InvalidConfiguration {
96            field: "index",
97            reason: "must fit in LibTorch's signed index range".to_owned(),
98        })?;
99        self.tensors
100            .iter()
101            .map(|tensor| tensor.f_get(index).map_err(RustTorchError::from))
102            .collect()
103    }
104}
105
106/// Tensor dataset with private backing storage and independently owned rows.
107pub struct ReplaySafeTensorDataset {
108    tensors: Vec<Tensor>,
109    len: usize,
110}
111
112impl Dataset for ReplaySafeTensorDataset {
113    type Sample = Vec<Tensor>;
114    type Error = RustTorchError;
115
116    fn len(&self) -> usize {
117        self.len
118    }
119
120    fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error> {
121        let index = i64::try_from(index).map_err(|_| RustTorchError::InvalidConfiguration {
122            field: "index",
123            reason: "must fit in LibTorch's signed index range".to_owned(),
124        })?;
125        self.tensors
126            .iter()
127            .map(|tensor| {
128                let row = tensor.f_get(index).map_err(RustTorchError::from)?;
129                deep_copy_tensor(&row)
130            })
131            .collect()
132    }
133}
134
135impl ReplaySafeDataset for ReplaySafeTensorDataset {}
136
137impl DatasetCheckpoint for ReplaySafeTensorDataset {
138    type State = ();
139
140    fn snapshot_dataset(&self) -> Self::State {}
141
142    fn validate_dataset_state(&self, _state: &Self::State) -> Result<()> {
143        Ok(())
144    }
145
146    fn restore_dataset_validated(&mut self, _state: &Self::State) {}
147}
148
149fn deep_copy_tensor(tensor: &Tensor) -> Result<Tensor> {
150    let mut copied = tensor.f_empty_like().map_err(RustTorchError::from)?;
151    copied.f_copy_(tensor).map_err(RustTorchError::from)?;
152    Ok(copied)
153}
154
155/// Explicit exact-checkpoint adapter for a replay-safe map dataset.
156#[derive(Clone, Copy, Debug, Default)]
157pub struct ReplaySafeMap<D> {
158    dataset: D,
159}
160
161impl<D> ReplaySafeMap<D> {
162    /// Wraps a dataset that explicitly implements [`ReplaySafeDataset`].
163    pub fn new(dataset: D) -> Self {
164        Self { dataset }
165    }
166
167    /// Returns the wrapped dataset.
168    pub fn into_inner(self) -> D {
169        self.dataset
170    }
171}
172
173impl<D> Dataset for ReplaySafeMap<D>
174where
175    D: ReplaySafeDataset,
176{
177    type Sample = D::Sample;
178    type Error = D::Error;
179
180    fn len(&self) -> usize {
181        self.dataset.len()
182    }
183
184    fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error> {
185        self.dataset.get(index)
186    }
187
188    fn get_batch(&self, indices: &[usize]) -> std::result::Result<Vec<Self::Sample>, Self::Error> {
189        self.dataset.get_batch(indices)
190    }
191
192    fn get_batch_with_context(
193        &self,
194        indices: &[usize],
195        context: &WorkerContext,
196    ) -> std::result::Result<Vec<Self::Sample>, Self::Error> {
197        self.dataset.get_batch_with_context(indices, context)
198    }
199}
200
201impl<D> ReplaySafeDataset for ReplaySafeMap<D> where D: ReplaySafeDataset {}
202
203impl<D> DatasetCheckpoint for ReplaySafeMap<D>
204where
205    D: ReplaySafeDataset,
206{
207    type State = ();
208
209    fn snapshot_dataset(&self) -> Self::State {}
210
211    fn validate_dataset_state(&self, _state: &Self::State) -> Result<()> {
212        Ok(())
213    }
214
215    fn restore_dataset_validated(&mut self, _state: &Self::State) {}
216}
217
218/// Explicit serial exact-checkpoint adapter for a transactional dataset.
219#[derive(Clone, Copy, Debug, Default)]
220pub struct TransactionalMap<D> {
221    dataset: D,
222}
223
224impl<D> TransactionalMap<D> {
225    /// Wraps a dataset that implements [`Checkpointable`].
226    pub fn new(dataset: D) -> Self {
227        Self { dataset }
228    }
229
230    /// Returns the wrapped dataset.
231    pub fn into_inner(self) -> D {
232        self.dataset
233    }
234}
235
236impl<D> Dataset for TransactionalMap<D>
237where
238    D: Dataset + Checkpointable,
239{
240    type Sample = D::Sample;
241    type Error = D::Error;
242
243    fn len(&self) -> usize {
244        self.dataset.len()
245    }
246
247    fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error> {
248        self.dataset.get(index)
249    }
250
251    fn get_batch(&self, indices: &[usize]) -> std::result::Result<Vec<Self::Sample>, Self::Error> {
252        self.dataset.get_batch(indices)
253    }
254
255    fn get_batch_with_context(
256        &self,
257        indices: &[usize],
258        context: &WorkerContext,
259    ) -> std::result::Result<Vec<Self::Sample>, Self::Error> {
260        self.dataset.get_batch_with_context(indices, context)
261    }
262}
263
264impl<D> DatasetCheckpoint for TransactionalMap<D>
265where
266    D: Dataset + Checkpointable,
267{
268    type State = D::State;
269
270    fn snapshot_dataset(&self) -> Self::State {
271        self.dataset.save_state()
272    }
273
274    fn validate_dataset_state(&self, state: &Self::State) -> Result<()> {
275        self.dataset.validate_state(state)
276    }
277
278    fn restore_dataset_validated(&mut self, state: &Self::State) {
279        self.dataset.load_validated(state);
280    }
281}
282
283/// A dataset that returns aligned samples from a tuple of child datasets.
284pub struct StackDataset<T> {
285    datasets: T,
286    len: usize,
287}
288
289mod sealed {
290    pub trait Sealed {}
291}
292
293/// A sealed tuple shape accepted by [`StackDataset`].
294///
295/// RustTorch implements this trait for dataset tuples with arities two
296/// through eight whose children share one error type.
297pub trait StackTuple: sealed::Sealed {
298    /// The tuple of child samples.
299    type Sample;
300    /// The error shared by every child dataset.
301    type Error;
302    /// Returns the child lengths for constructor validation.
303    #[doc(hidden)]
304    fn lengths(&self) -> Vec<usize>;
305    /// Loads the aligned child samples at `index`.
306    #[doc(hidden)]
307    fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error>;
308}
309
310impl<T> StackDataset<T>
311where
312    T: StackTuple,
313{
314    /// Creates a stack from child datasets with equal lengths.
315    ///
316    /// # Errors
317    ///
318    /// Returns [`RustTorchError::InvalidDimensions`] when a child length
319    /// differs from the first child's length.
320    pub fn new(datasets: T) -> Result<Self> {
321        let lengths = datasets.lengths();
322        let len = lengths[0];
323        if lengths.iter().any(|&child_len| child_len != len) {
324            return Err(RustTorchError::InvalidDimensions {
325                context: "StackDataset child lengths".to_owned(),
326                expected: format!("all lengths equal to {len}"),
327                actual: format!("{lengths:?}"),
328            });
329        }
330        Ok(Self { datasets, len })
331    }
332}
333
334impl<T> Dataset for StackDataset<T>
335where
336    T: StackTuple,
337{
338    type Sample = T::Sample;
339    type Error = T::Error;
340
341    fn len(&self) -> usize {
342        self.len
343    }
344
345    fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error> {
346        StackTuple::get(&self.datasets, index)
347    }
348}
349
350macro_rules! impl_stack_dataset {
351    ($First:ident:$first:tt, $($Dataset:ident:$field:tt),+) => {
352        impl<$First, $($Dataset),+> sealed::Sealed for ($First, $($Dataset,)+)
353        where
354            $First: Dataset,
355            $($Dataset: Dataset<Error = $First::Error>),+
356        {}
357
358        impl<$First, $($Dataset),+> StackTuple for ($First, $($Dataset,)+)
359        where
360            $First: Dataset,
361            $($Dataset: Dataset<Error = $First::Error>),+
362        {
363            type Sample = ($First::Sample, $($Dataset::Sample,)+);
364            type Error = $First::Error;
365
366            fn lengths(&self) -> Vec<usize> {
367                vec![self.$first.len(), $(self.$field.len()),+]
368            }
369
370            fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error> {
371                Ok((self.$first.get(index)?, $(self.$field.get(index)?,)+))
372            }
373        }
374    };
375}
376
377impl_stack_dataset!(A:0, B:1);
378impl_stack_dataset!(A:0, B:1, C:2);
379impl_stack_dataset!(A:0, B:1, C:2, D:3);
380impl_stack_dataset!(A:0, B:1, C:2, D:3, E:4);
381impl_stack_dataset!(A:0, B:1, C:2, D:3, E:4, F:5);
382impl_stack_dataset!(A:0, B:1, C:2, D:3, E:4, F:5, G:6);
383impl_stack_dataset!(A:0, B:1, C:2, D:3, E:4, F:5, G:6, H:7);
384
385trait ReplaySafeStackTuple {}
386
387macro_rules! impl_replay_safe_stack_tuple {
388    ($($type:ident),+ $(,)?) => {
389        impl<$($type),+> ReplaySafeStackTuple for ($($type,)+)
390        where
391            $($type: ReplaySafeDataset),+
392        {}
393    };
394}
395
396impl_replay_safe_stack_tuple!(A, B);
397impl_replay_safe_stack_tuple!(A, B, C);
398impl_replay_safe_stack_tuple!(A, B, C, D);
399impl_replay_safe_stack_tuple!(A, B, C, D, E);
400impl_replay_safe_stack_tuple!(A, B, C, D, E, F);
401impl_replay_safe_stack_tuple!(A, B, C, D, E, F, G);
402impl_replay_safe_stack_tuple!(A, B, C, D, E, F, G, H);
403
404impl<T> ReplaySafeDataset for StackDataset<T> where T: StackTuple + ReplaySafeStackTuple {}
405
406/// A dataset that maps one global index across consecutive child datasets.
407pub struct ConcatDataset<D> {
408    datasets: Vec<D>,
409    cumulative_sizes: Vec<usize>,
410}
411
412impl<D> ConcatDataset<D>
413where
414    D: Dataset,
415{
416    /// Creates a concatenation of one or more child datasets.
417    ///
418    /// # Errors
419    ///
420    /// Returns [`RustTorchError::InvalidConfiguration`] when `datasets` is
421    /// empty or their combined length overflows [`usize`].
422    pub fn new(datasets: Vec<D>) -> Result<Self> {
423        if datasets.is_empty() {
424            return Err(RustTorchError::InvalidConfiguration {
425                field: "datasets",
426                reason: "must not be empty".to_owned(),
427            });
428        }
429        let mut total = 0usize;
430        let cumulative_sizes = datasets
431            .iter()
432            .map(|dataset| {
433                total = total.checked_add(dataset.len()).ok_or_else(|| {
434                    RustTorchError::InvalidConfiguration {
435                        field: "datasets",
436                        reason: "combined length overflows usize".to_owned(),
437                    }
438                })?;
439                Ok(total)
440            })
441            .collect::<Result<Vec<_>>>()?;
442        Ok(Self {
443            datasets,
444            cumulative_sizes,
445        })
446    }
447}
448
449impl<D> Dataset for ConcatDataset<D>
450where
451    D: Dataset,
452{
453    type Sample = D::Sample;
454    type Error = D::Error;
455
456    fn len(&self) -> usize {
457        self.cumulative_sizes.last().copied().unwrap_or(0)
458    }
459
460    fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error> {
461        let dataset_index = self
462            .cumulative_sizes
463            .partition_point(|&cumulative_size| cumulative_size <= index);
464        let previous_size = dataset_index
465            .checked_sub(1)
466            .map_or(0, |previous| self.cumulative_sizes[previous]);
467        self.datasets[dataset_index].get(index - previous_size)
468    }
469}
470
471impl<D> ReplaySafeDataset for ConcatDataset<D> where D: ReplaySafeDataset {}
472
473/// A dataset containing selected indices from another dataset.
474pub struct Subset<D> {
475    dataset: D,
476    indices: Vec<usize>,
477}
478
479impl<D> Subset<D>
480where
481    D: Dataset,
482{
483    /// Creates a subset after validating every source index.
484    ///
485    /// # Errors
486    ///
487    /// Returns [`RustTorchError::InvalidConfiguration`] when an index is not
488    /// less than the source dataset length.
489    pub fn new(dataset: D, indices: Vec<usize>) -> Result<Self> {
490        if let Some(&index) = indices.iter().find(|&&index| index >= dataset.len()) {
491            return Err(RustTorchError::InvalidConfiguration {
492                field: "indices",
493                reason: format!("index {index} is out of range for length {}", dataset.len()),
494            });
495        }
496        Ok(Self { dataset, indices })
497    }
498}
499
500impl<D> Dataset for Subset<D>
501where
502    D: Dataset,
503{
504    type Sample = D::Sample;
505    type Error = D::Error;
506
507    fn len(&self) -> usize {
508        self.indices.len()
509    }
510
511    fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error> {
512        self.dataset.get(self.indices[index])
513    }
514}
515
516impl<D> ReplaySafeDataset for Subset<D> where D: ReplaySafeDataset {}
517
518/// One requested output length for [`random_split`].
519#[derive(Clone, Copy, Debug, PartialEq)]
520pub enum SplitLength {
521    /// An exact sample count.
522    Count(usize),
523    /// A fraction of the source length; all fractions must sum to one.
524    Fraction(f64),
525}
526
527/// Randomly partitions a shared dataset into non-overlapping subsets.
528///
529/// Count lengths must sum exactly to the dataset length. Fractional lengths
530/// are floored, then leftover samples are assigned round-robin from the first
531/// split. The seeded permutation is reproducible within this implementation.
532///
533/// # Errors
534///
535/// Returns [`RustTorchError::InvalidConfiguration`] for mixed length kinds,
536/// invalid fractions, incorrect totals, or count overflow.
537pub fn random_split<D>(
538    dataset: Arc<D>,
539    lengths: &[SplitLength],
540    seed: u64,
541) -> Result<Vec<Subset<Arc<D>>>>
542where
543    D: Dataset,
544{
545    let counts = match lengths.first() {
546        None | Some(SplitLength::Count(_)) => {
547            if lengths
548                .iter()
549                .any(|length| matches!(length, SplitLength::Fraction(_)))
550            {
551                return Err(invalid_split("count and fraction lengths cannot be mixed"));
552            }
553            let counts = lengths
554                .iter()
555                .map(|length| match length {
556                    SplitLength::Count(count) => Ok(*count),
557                    SplitLength::Fraction(_) => unreachable!(),
558                })
559                .collect::<Result<Vec<_>>>()?;
560            let total = counts.iter().try_fold(0usize, |total, &count| {
561                total
562                    .checked_add(count)
563                    .ok_or_else(|| invalid_split("count total overflows usize"))
564            })?;
565            if total != dataset.len() {
566                return Err(invalid_split(&format!(
567                    "counts sum to {total}, expected {}",
568                    dataset.len()
569                )));
570            }
571            counts
572        }
573        Some(SplitLength::Fraction(_)) => {
574            let fractions = lengths
575                .iter()
576                .map(|length| match length {
577                    SplitLength::Fraction(fraction)
578                        if fraction.is_finite() && (0.0..=1.0).contains(fraction) =>
579                    {
580                        Ok(*fraction)
581                    }
582                    SplitLength::Fraction(_) => Err(invalid_split(
583                        "fractions must be finite and between zero and one",
584                    )),
585                    SplitLength::Count(_) => {
586                        Err(invalid_split("count and fraction lengths cannot be mixed"))
587                    }
588                })
589                .collect::<Result<Vec<_>>>()?;
590            let total = fractions.iter().sum::<f64>();
591            if total > 1.0 || (total - 1.0).abs() > 1e-9 {
592                return Err(invalid_split(&format!(
593                    "fractions sum to {total}, expected 1"
594                )));
595            }
596            let mut counts = fractions
597                .iter()
598                .map(|fraction| (fraction * dataset.len() as f64).floor() as usize)
599                .collect::<Vec<_>>();
600            let assigned = counts.iter().try_fold(0usize, |total, &count| {
601                total
602                    .checked_add(count)
603                    .ok_or_else(|| invalid_split("fractional count total overflows usize"))
604            })?;
605            let remainder = dataset.len().checked_sub(assigned).ok_or_else(|| {
606                invalid_split("fractional floors exceed the source dataset length")
607            })?;
608            let split_count = counts.len();
609            for index in 0..remainder {
610                counts[index % split_count] += 1;
611            }
612            counts
613        }
614    };
615
616    let mut indices = (0..dataset.len()).collect::<Vec<_>>();
617    indices.shuffle(&mut ChaCha12Rng::seed_from_u64(seed));
618    let mut offset = 0;
619    counts
620        .into_iter()
621        .map(|count| {
622            let next = offset + count;
623            let subset = Subset::new(Arc::clone(&dataset), indices[offset..next].to_vec());
624            offset = next;
625            subset
626        })
627        .collect()
628}
629
630fn invalid_split(reason: &str) -> RustTorchError {
631    RustTorchError::InvalidConfiguration {
632        field: "lengths",
633        reason: reason.to_owned(),
634    }
635}
636
637/// Flattens datasets or other iterables using the standard iterator adapter.
638pub fn chain_datasets<I>(datasets: I) -> impl Iterator<Item = <I::Item as IntoIterator>::Item>
639where
640    I: IntoIterator,
641    I::Item: IntoIterator,
642{
643    datasets.into_iter().flatten()
644}
645
646impl<D> Dataset for Arc<D>
647where
648    D: Dataset + ?Sized,
649{
650    type Sample = D::Sample;
651    type Error = D::Error;
652
653    fn len(&self) -> usize {
654        self.as_ref().len()
655    }
656
657    fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error> {
658        self.as_ref().get(index)
659    }
660
661    fn get_batch(&self, indices: &[usize]) -> std::result::Result<Vec<Self::Sample>, Self::Error> {
662        self.as_ref().get_batch(indices)
663    }
664
665    fn get_batch_with_context(
666        &self,
667        indices: &[usize],
668        context: &WorkerContext,
669    ) -> std::result::Result<Vec<Self::Sample>, Self::Error> {
670        self.as_ref().get_batch_with_context(indices, context)
671    }
672}
673
674impl<D> ReplaySafeDataset for Arc<D> where D: ReplaySafeDataset + ?Sized {}