torsh-data 0.1.0

Data loading and preprocessing utilities for ToRSh
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49

//! # MemoryMappedDataset - Trait Implementations
//!
//! This module contains trait implementations for `MemoryMappedDataset`.
//!
//! ## Implemented Traits
//!
//! - `Dataset`
//!
//! 🤖 Generated with [SplitRS](https://github.com/cool-japan/splitrs)

use torsh_core::error::Result;
use torsh_tensor::Tensor;

use super::functions::Dataset;
use super::types::MemoryMappedDataset;

#[cfg(all(feature = "std", feature = "mmap-support"))]
impl<T: torsh_core::dtype::TensorElement> Dataset for MemoryMappedDataset<T> {
    type Item = Tensor<T>;
    fn len(&self) -> usize {
        self.length
    }
    fn get(&self, index: usize) -> Result<Self::Item> {
        if index >= self.length {
            return Err(torsh_core::error::TorshError::IndexError {
                index,
                size: self.length,
            });
        }
        if index >= self.tensor_offsets.len() || index >= self.tensor_shapes.len() {
            return Err(torsh_core::error::TorshError::InvalidArgument(
                "Invalid tensor metadata".to_string(),
            ));
        }
        let offset = self.tensor_offsets[index];
        let shape = &self.tensor_shapes[index];
        unsafe {
            let data_ptr = self.mmap.as_ptr().add(offset) as *const T;
            let numel = shape.iter().product::<usize>();
            let data_slice = std::slice::from_raw_parts(data_ptr, numel);
            let data_vec = data_slice.to_vec();
            Tensor::from_data(
                data_vec,
                shape.to_vec(),
                torsh_core::device::DeviceType::Cpu,
            )
        }
    }
}