1use std::sync::Arc;
2
3use rand::{SeedableRng, seq::SliceRandom};
4use rand_chacha::ChaCha12Rng;
5use rusttorch_core::{Result, RustTorchError, Tensor};
6
7use crate::{Checkpointable, Dataset, DatasetCheckpoint, ReplaySafeDataset, WorkerContext};
8
9pub struct TensorDataset {
14 tensors: Vec<Tensor>,
15 len: usize,
16}
17
18impl TensorDataset {
19 pub fn new(tensors: Vec<Tensor>) -> Result<Self> {
27 let first = tensors
28 .first()
29 .ok_or_else(|| RustTorchError::InvalidConfiguration {
30 field: "tensors",
31 reason: "must not be empty".to_owned(),
32 })?;
33 let first_size = first.size();
34 let len = first_size
35 .first()
36 .copied()
37 .ok_or_else(|| RustTorchError::InvalidDimensions {
38 context: "TensorDataset tensor 0".to_owned(),
39 expected: "at least one dimension".to_owned(),
40 actual: format!("{first_size:?}"),
41 })?;
42
43 for (index, tensor) in tensors.iter().enumerate().skip(1) {
44 let size = tensor.size();
45 if size.first().copied() != Some(len) {
46 return Err(RustTorchError::InvalidDimensions {
47 context: format!("TensorDataset tensor {index}"),
48 expected: format!("first dimension {len}"),
49 actual: format!("{size:?}"),
50 });
51 }
52 }
53
54 Ok(Self {
55 tensors,
56 len: usize::try_from(len).map_err(|_| RustTorchError::InvalidDimensions {
57 context: "TensorDataset tensor 0".to_owned(),
58 expected: "a nonnegative first dimension representable as usize".to_owned(),
59 actual: len.to_string(),
60 })?,
61 })
62 }
63
64 pub fn into_replay_safe(self) -> Result<ReplaySafeTensorDataset> {
74 let tensors = self
75 .tensors
76 .iter()
77 .map(deep_copy_tensor)
78 .collect::<Result<Vec<_>>>()?;
79 Ok(ReplaySafeTensorDataset {
80 tensors,
81 len: self.len,
82 })
83 }
84}
85
86impl Dataset for TensorDataset {
87 type Sample = Vec<Tensor>;
88 type Error = RustTorchError;
89
90 fn len(&self) -> usize {
91 self.len
92 }
93
94 fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error> {
95 let index = i64::try_from(index).map_err(|_| RustTorchError::InvalidConfiguration {
96 field: "index",
97 reason: "must fit in LibTorch's signed index range".to_owned(),
98 })?;
99 self.tensors
100 .iter()
101 .map(|tensor| tensor.f_get(index).map_err(RustTorchError::from))
102 .collect()
103 }
104}
105
106pub struct ReplaySafeTensorDataset {
108 tensors: Vec<Tensor>,
109 len: usize,
110}
111
112impl Dataset for ReplaySafeTensorDataset {
113 type Sample = Vec<Tensor>;
114 type Error = RustTorchError;
115
116 fn len(&self) -> usize {
117 self.len
118 }
119
120 fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error> {
121 let index = i64::try_from(index).map_err(|_| RustTorchError::InvalidConfiguration {
122 field: "index",
123 reason: "must fit in LibTorch's signed index range".to_owned(),
124 })?;
125 self.tensors
126 .iter()
127 .map(|tensor| {
128 let row = tensor.f_get(index).map_err(RustTorchError::from)?;
129 deep_copy_tensor(&row)
130 })
131 .collect()
132 }
133}
134
135impl ReplaySafeDataset for ReplaySafeTensorDataset {}
136
137impl DatasetCheckpoint for ReplaySafeTensorDataset {
138 type State = ();
139
140 fn snapshot_dataset(&self) -> Self::State {}
141
142 fn validate_dataset_state(&self, _state: &Self::State) -> Result<()> {
143 Ok(())
144 }
145
146 fn restore_dataset_validated(&mut self, _state: &Self::State) {}
147}
148
149fn deep_copy_tensor(tensor: &Tensor) -> Result<Tensor> {
150 let mut copied = tensor.f_empty_like().map_err(RustTorchError::from)?;
151 copied.f_copy_(tensor).map_err(RustTorchError::from)?;
152 Ok(copied)
153}
154
155#[derive(Clone, Copy, Debug, Default)]
157pub struct ReplaySafeMap<D> {
158 dataset: D,
159}
160
161impl<D> ReplaySafeMap<D> {
162 pub fn new(dataset: D) -> Self {
164 Self { dataset }
165 }
166
167 pub fn into_inner(self) -> D {
169 self.dataset
170 }
171}
172
173impl<D> Dataset for ReplaySafeMap<D>
174where
175 D: ReplaySafeDataset,
176{
177 type Sample = D::Sample;
178 type Error = D::Error;
179
180 fn len(&self) -> usize {
181 self.dataset.len()
182 }
183
184 fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error> {
185 self.dataset.get(index)
186 }
187
188 fn get_batch(&self, indices: &[usize]) -> std::result::Result<Vec<Self::Sample>, Self::Error> {
189 self.dataset.get_batch(indices)
190 }
191
192 fn get_batch_with_context(
193 &self,
194 indices: &[usize],
195 context: &WorkerContext,
196 ) -> std::result::Result<Vec<Self::Sample>, Self::Error> {
197 self.dataset.get_batch_with_context(indices, context)
198 }
199}
200
201impl<D> ReplaySafeDataset for ReplaySafeMap<D> where D: ReplaySafeDataset {}
202
203impl<D> DatasetCheckpoint for ReplaySafeMap<D>
204where
205 D: ReplaySafeDataset,
206{
207 type State = ();
208
209 fn snapshot_dataset(&self) -> Self::State {}
210
211 fn validate_dataset_state(&self, _state: &Self::State) -> Result<()> {
212 Ok(())
213 }
214
215 fn restore_dataset_validated(&mut self, _state: &Self::State) {}
216}
217
218#[derive(Clone, Copy, Debug, Default)]
220pub struct TransactionalMap<D> {
221 dataset: D,
222}
223
224impl<D> TransactionalMap<D> {
225 pub fn new(dataset: D) -> Self {
227 Self { dataset }
228 }
229
230 pub fn into_inner(self) -> D {
232 self.dataset
233 }
234}
235
236impl<D> Dataset for TransactionalMap<D>
237where
238 D: Dataset + Checkpointable,
239{
240 type Sample = D::Sample;
241 type Error = D::Error;
242
243 fn len(&self) -> usize {
244 self.dataset.len()
245 }
246
247 fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error> {
248 self.dataset.get(index)
249 }
250
251 fn get_batch(&self, indices: &[usize]) -> std::result::Result<Vec<Self::Sample>, Self::Error> {
252 self.dataset.get_batch(indices)
253 }
254
255 fn get_batch_with_context(
256 &self,
257 indices: &[usize],
258 context: &WorkerContext,
259 ) -> std::result::Result<Vec<Self::Sample>, Self::Error> {
260 self.dataset.get_batch_with_context(indices, context)
261 }
262}
263
264impl<D> DatasetCheckpoint for TransactionalMap<D>
265where
266 D: Dataset + Checkpointable,
267{
268 type State = D::State;
269
270 fn snapshot_dataset(&self) -> Self::State {
271 self.dataset.save_state()
272 }
273
274 fn validate_dataset_state(&self, state: &Self::State) -> Result<()> {
275 self.dataset.validate_state(state)
276 }
277
278 fn restore_dataset_validated(&mut self, state: &Self::State) {
279 self.dataset.load_validated(state);
280 }
281}
282
283pub struct StackDataset<T> {
285 datasets: T,
286 len: usize,
287}
288
289mod sealed {
290 pub trait Sealed {}
291}
292
293pub trait StackTuple: sealed::Sealed {
298 type Sample;
300 type Error;
302 #[doc(hidden)]
304 fn lengths(&self) -> Vec<usize>;
305 #[doc(hidden)]
307 fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error>;
308}
309
310impl<T> StackDataset<T>
311where
312 T: StackTuple,
313{
314 pub fn new(datasets: T) -> Result<Self> {
321 let lengths = datasets.lengths();
322 let len = lengths[0];
323 if lengths.iter().any(|&child_len| child_len != len) {
324 return Err(RustTorchError::InvalidDimensions {
325 context: "StackDataset child lengths".to_owned(),
326 expected: format!("all lengths equal to {len}"),
327 actual: format!("{lengths:?}"),
328 });
329 }
330 Ok(Self { datasets, len })
331 }
332}
333
334impl<T> Dataset for StackDataset<T>
335where
336 T: StackTuple,
337{
338 type Sample = T::Sample;
339 type Error = T::Error;
340
341 fn len(&self) -> usize {
342 self.len
343 }
344
345 fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error> {
346 StackTuple::get(&self.datasets, index)
347 }
348}
349
350macro_rules! impl_stack_dataset {
351 ($First:ident:$first:tt, $($Dataset:ident:$field:tt),+) => {
352 impl<$First, $($Dataset),+> sealed::Sealed for ($First, $($Dataset,)+)
353 where
354 $First: Dataset,
355 $($Dataset: Dataset<Error = $First::Error>),+
356 {}
357
358 impl<$First, $($Dataset),+> StackTuple for ($First, $($Dataset,)+)
359 where
360 $First: Dataset,
361 $($Dataset: Dataset<Error = $First::Error>),+
362 {
363 type Sample = ($First::Sample, $($Dataset::Sample,)+);
364 type Error = $First::Error;
365
366 fn lengths(&self) -> Vec<usize> {
367 vec![self.$first.len(), $(self.$field.len()),+]
368 }
369
370 fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error> {
371 Ok((self.$first.get(index)?, $(self.$field.get(index)?,)+))
372 }
373 }
374 };
375}
376
377impl_stack_dataset!(A:0, B:1);
378impl_stack_dataset!(A:0, B:1, C:2);
379impl_stack_dataset!(A:0, B:1, C:2, D:3);
380impl_stack_dataset!(A:0, B:1, C:2, D:3, E:4);
381impl_stack_dataset!(A:0, B:1, C:2, D:3, E:4, F:5);
382impl_stack_dataset!(A:0, B:1, C:2, D:3, E:4, F:5, G:6);
383impl_stack_dataset!(A:0, B:1, C:2, D:3, E:4, F:5, G:6, H:7);
384
385trait ReplaySafeStackTuple {}
386
387macro_rules! impl_replay_safe_stack_tuple {
388 ($($type:ident),+ $(,)?) => {
389 impl<$($type),+> ReplaySafeStackTuple for ($($type,)+)
390 where
391 $($type: ReplaySafeDataset),+
392 {}
393 };
394}
395
396impl_replay_safe_stack_tuple!(A, B);
397impl_replay_safe_stack_tuple!(A, B, C);
398impl_replay_safe_stack_tuple!(A, B, C, D);
399impl_replay_safe_stack_tuple!(A, B, C, D, E);
400impl_replay_safe_stack_tuple!(A, B, C, D, E, F);
401impl_replay_safe_stack_tuple!(A, B, C, D, E, F, G);
402impl_replay_safe_stack_tuple!(A, B, C, D, E, F, G, H);
403
404impl<T> ReplaySafeDataset for StackDataset<T> where T: StackTuple + ReplaySafeStackTuple {}
405
406pub struct ConcatDataset<D> {
408 datasets: Vec<D>,
409 cumulative_sizes: Vec<usize>,
410}
411
412impl<D> ConcatDataset<D>
413where
414 D: Dataset,
415{
416 pub fn new(datasets: Vec<D>) -> Result<Self> {
423 if datasets.is_empty() {
424 return Err(RustTorchError::InvalidConfiguration {
425 field: "datasets",
426 reason: "must not be empty".to_owned(),
427 });
428 }
429 let mut total = 0usize;
430 let cumulative_sizes = datasets
431 .iter()
432 .map(|dataset| {
433 total = total.checked_add(dataset.len()).ok_or_else(|| {
434 RustTorchError::InvalidConfiguration {
435 field: "datasets",
436 reason: "combined length overflows usize".to_owned(),
437 }
438 })?;
439 Ok(total)
440 })
441 .collect::<Result<Vec<_>>>()?;
442 Ok(Self {
443 datasets,
444 cumulative_sizes,
445 })
446 }
447}
448
449impl<D> Dataset for ConcatDataset<D>
450where
451 D: Dataset,
452{
453 type Sample = D::Sample;
454 type Error = D::Error;
455
456 fn len(&self) -> usize {
457 self.cumulative_sizes.last().copied().unwrap_or(0)
458 }
459
460 fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error> {
461 let dataset_index = self
462 .cumulative_sizes
463 .partition_point(|&cumulative_size| cumulative_size <= index);
464 let previous_size = dataset_index
465 .checked_sub(1)
466 .map_or(0, |previous| self.cumulative_sizes[previous]);
467 self.datasets[dataset_index].get(index - previous_size)
468 }
469}
470
471impl<D> ReplaySafeDataset for ConcatDataset<D> where D: ReplaySafeDataset {}
472
473pub struct Subset<D> {
475 dataset: D,
476 indices: Vec<usize>,
477}
478
479impl<D> Subset<D>
480where
481 D: Dataset,
482{
483 pub fn new(dataset: D, indices: Vec<usize>) -> Result<Self> {
490 if let Some(&index) = indices.iter().find(|&&index| index >= dataset.len()) {
491 return Err(RustTorchError::InvalidConfiguration {
492 field: "indices",
493 reason: format!("index {index} is out of range for length {}", dataset.len()),
494 });
495 }
496 Ok(Self { dataset, indices })
497 }
498}
499
500impl<D> Dataset for Subset<D>
501where
502 D: Dataset,
503{
504 type Sample = D::Sample;
505 type Error = D::Error;
506
507 fn len(&self) -> usize {
508 self.indices.len()
509 }
510
511 fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error> {
512 self.dataset.get(self.indices[index])
513 }
514}
515
516impl<D> ReplaySafeDataset for Subset<D> where D: ReplaySafeDataset {}
517
518#[derive(Clone, Copy, Debug, PartialEq)]
520pub enum SplitLength {
521 Count(usize),
523 Fraction(f64),
525}
526
527pub fn random_split<D>(
538 dataset: Arc<D>,
539 lengths: &[SplitLength],
540 seed: u64,
541) -> Result<Vec<Subset<Arc<D>>>>
542where
543 D: Dataset,
544{
545 let counts = match lengths.first() {
546 None | Some(SplitLength::Count(_)) => {
547 if lengths
548 .iter()
549 .any(|length| matches!(length, SplitLength::Fraction(_)))
550 {
551 return Err(invalid_split("count and fraction lengths cannot be mixed"));
552 }
553 let counts = lengths
554 .iter()
555 .map(|length| match length {
556 SplitLength::Count(count) => Ok(*count),
557 SplitLength::Fraction(_) => unreachable!(),
558 })
559 .collect::<Result<Vec<_>>>()?;
560 let total = counts.iter().try_fold(0usize, |total, &count| {
561 total
562 .checked_add(count)
563 .ok_or_else(|| invalid_split("count total overflows usize"))
564 })?;
565 if total != dataset.len() {
566 return Err(invalid_split(&format!(
567 "counts sum to {total}, expected {}",
568 dataset.len()
569 )));
570 }
571 counts
572 }
573 Some(SplitLength::Fraction(_)) => {
574 let fractions = lengths
575 .iter()
576 .map(|length| match length {
577 SplitLength::Fraction(fraction)
578 if fraction.is_finite() && (0.0..=1.0).contains(fraction) =>
579 {
580 Ok(*fraction)
581 }
582 SplitLength::Fraction(_) => Err(invalid_split(
583 "fractions must be finite and between zero and one",
584 )),
585 SplitLength::Count(_) => {
586 Err(invalid_split("count and fraction lengths cannot be mixed"))
587 }
588 })
589 .collect::<Result<Vec<_>>>()?;
590 let total = fractions.iter().sum::<f64>();
591 if total > 1.0 || (total - 1.0).abs() > 1e-9 {
592 return Err(invalid_split(&format!(
593 "fractions sum to {total}, expected 1"
594 )));
595 }
596 let mut counts = fractions
597 .iter()
598 .map(|fraction| (fraction * dataset.len() as f64).floor() as usize)
599 .collect::<Vec<_>>();
600 let assigned = counts.iter().try_fold(0usize, |total, &count| {
601 total
602 .checked_add(count)
603 .ok_or_else(|| invalid_split("fractional count total overflows usize"))
604 })?;
605 let remainder = dataset.len().checked_sub(assigned).ok_or_else(|| {
606 invalid_split("fractional floors exceed the source dataset length")
607 })?;
608 let split_count = counts.len();
609 for index in 0..remainder {
610 counts[index % split_count] += 1;
611 }
612 counts
613 }
614 };
615
616 let mut indices = (0..dataset.len()).collect::<Vec<_>>();
617 indices.shuffle(&mut ChaCha12Rng::seed_from_u64(seed));
618 let mut offset = 0;
619 counts
620 .into_iter()
621 .map(|count| {
622 let next = offset + count;
623 let subset = Subset::new(Arc::clone(&dataset), indices[offset..next].to_vec());
624 offset = next;
625 subset
626 })
627 .collect()
628}
629
630fn invalid_split(reason: &str) -> RustTorchError {
631 RustTorchError::InvalidConfiguration {
632 field: "lengths",
633 reason: reason.to_owned(),
634 }
635}
636
637pub fn chain_datasets<I>(datasets: I) -> impl Iterator<Item = <I::Item as IntoIterator>::Item>
639where
640 I: IntoIterator,
641 I::Item: IntoIterator,
642{
643 datasets.into_iter().flatten()
644}
645
646impl<D> Dataset for Arc<D>
647where
648 D: Dataset + ?Sized,
649{
650 type Sample = D::Sample;
651 type Error = D::Error;
652
653 fn len(&self) -> usize {
654 self.as_ref().len()
655 }
656
657 fn get(&self, index: usize) -> std::result::Result<Self::Sample, Self::Error> {
658 self.as_ref().get(index)
659 }
660
661 fn get_batch(&self, indices: &[usize]) -> std::result::Result<Vec<Self::Sample>, Self::Error> {
662 self.as_ref().get_batch(indices)
663 }
664
665 fn get_batch_with_context(
666 &self,
667 indices: &[usize],
668 context: &WorkerContext,
669 ) -> std::result::Result<Vec<Self::Sample>, Self::Error> {
670 self.as_ref().get_batch_with_context(indices, context)
671 }
672}
673
674impl<D> ReplaySafeDataset for Arc<D> where D: ReplaySafeDataset + ?Sized {}