tenshift-core 0.1.3

Thread-safe, backpressure-aware data loading pipeline for iterative processing
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

use super::tensor::DType;
use crate::error::{Error, Result};
use std::sync::OnceLock;

pub(crate) fn expected_byte_len(dtype: DType, shape: &[usize]) -> Result<usize> {
    let elements = shape.iter().try_fold(1_usize, |acc, &dim| {
        acc.checked_mul(dim).ok_or_else(|| Error::InvalidConfig {
            reason: format!("tensor shape {shape:?} overflows usize element counting"),
        })
    })?;

    elements
        .checked_mul(dtype.element_size())
        .ok_or_else(|| Error::InvalidConfig {
            reason: format!(
                "tensor shape {shape:?} with dtype {dtype:?} overflows usize byte counting"
            ),
        })
}

pub(crate) trait ToLeBytesArray<const N: usize> {
    fn to_le_bytes_array(&self) -> [u8; N];
}

impl ToLeBytesArray<4> for f32 {
    fn to_le_bytes_array(&self) -> [u8; 4] {
        self.to_le_bytes()
    }
}

impl ToLeBytesArray<8> for f64 {
    fn to_le_bytes_array(&self) -> [u8; 8] {
        self.to_le_bytes()
    }
}

impl ToLeBytesArray<4> for i32 {
    fn to_le_bytes_array(&self) -> [u8; 4] {
        self.to_le_bytes()
    }
}

impl ToLeBytesArray<8> for i64 {
    fn to_le_bytes_array(&self) -> [u8; 8] {
        self.to_le_bytes()
    }
}

/// Bulk-convert a numeric slice to little-endian bytes.
pub(crate) fn numeric_to_le_bytes<T: Copy + ToLeBytesArray<N>, const N: usize>(
    data: &[T],
) -> Vec<u8> {
    let mut bytes = Vec::with_capacity(data.len() * N);
    for item in data {
        bytes.extend_from_slice(&item.to_le_bytes_array());
    }
    bytes
}

/// Safe reinterpret cast for aligned, correctly-sized data.
pub(crate) fn cast_numeric_slice<'a, T, const N: usize>(
    data: &'a [u8],
    cache: &'a OnceLock<Vec<T>>,
    decode: fn([u8; N]) -> T,
) -> Result<&'a [T]>
where
    T: Copy,
{
    let elem_size = std::mem::size_of::<T>();
    if data.len() % elem_size != 0 {
        return Err(Error::InvalidConfig {
            reason: format!(
                "data length {} is not a multiple of element size {}",
                data.len(),
                elem_size
            ),
        });
    }

    Ok(cache
        .get_or_init(|| {
            data.chunks_exact(N)
                .map(|chunk| {
                    let mut bytes = [0_u8; N];
                    bytes.copy_from_slice(chunk);
                    decode(bytes)
                })
                .collect()
        })
        .as_slice())
}