tetration 0.1.9

Chunked tensor .tet format, tet CLI, and optional C ABI (tetration-ffi)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189

//! Integer logical materialize (`i32` / `i64` / `u8` / `u16` / `i16`).

mod core;
mod fold;
#[macro_use]
mod macros;

use std::path::Path;

use crate::query::decode::chunk_decode;
use crate::query::{
    dispatch::accumulate_chunk_read_bytes,
    engine::spill_policy::TempSpillFile,
    types::{ReadPlan, TetError},
};
use crate::utils::{dtype::ElementDtype, i16_le, i32_le, i64_le, u8_le, u16_le, u32_le, u64_le};

use super::parallel;
use super::validate::validate_read_plan_geometry;

use core::{materialize_read_plan_int_le_core, spill_read_plan_int_le_impl};

pub(crate) use fold::IntVisit;
pub(crate) use fold::fold_read_plan_scalar_operation_int;

define_int_materialize! {
    i32;
    backing LogicalI32Backing;
    le_mod i32_le;
    scatter chunk_decode::scatter_chunk_into_plan_i32;
    scatter_seq scatter_fill_sequential_i32;
    scatter_ty ScatterI32Fn;
    scatter_par parallel::materialize_scatter_fill_parallel_i32;
    core_fn materialize_read_plan_i32_le_core;
    read_fn materialize_read_plan_i32_le;
    spill_fn spill_read_plan_i32_le;
    type_label "i32";
    into_vec_fn materialize_into_vec_i32;
    spill_file_fn preview_from_spill_file_i32;
    preview_mat_fn preview_from_materialized_i32;
    as_f64_fn materialized_logical_as_f64_i32;
    promote_inmem |x| f64::from(x);
    promote_spill |x| f64::from(x);
}

define_int_materialize! {
    i64;
    backing LogicalI64Backing;
    le_mod i64_le;
    scatter chunk_decode::scatter_chunk_into_plan_i64;
    scatter_seq scatter_fill_sequential_i64;
    scatter_ty ScatterI64Fn;
    scatter_par parallel::materialize_scatter_fill_parallel_i64;
    core_fn materialize_read_plan_i64_le_core;
    read_fn materialize_read_plan_i64_le;
    spill_fn spill_read_plan_i64_le;
    type_label "i64";
    into_vec_fn materialize_into_vec_i64;
    spill_file_fn preview_from_spill_file_i64;
    preview_mat_fn preview_from_materialized_i64;
    as_f64_fn materialized_logical_as_f64_i64;
    promote_inmem |x| x as f64;
    promote_spill |x| x as f64;
}

define_int_materialize! {
    u8;
    backing LogicalU8Backing;
    le_mod u8_le;
    scatter chunk_decode::scatter_chunk_into_plan_u8;
    scatter_seq scatter_fill_sequential_u8;
    scatter_ty ScatterU8Fn;
    scatter_par parallel::materialize_scatter_fill_parallel_u8;
    core_fn materialize_read_plan_u8_le_core;
    read_fn materialize_read_plan_u8_le;
    spill_fn spill_read_plan_u8_le;
    type_label "u8";
    into_vec_fn materialize_into_vec_u8;
    spill_file_fn preview_from_spill_file_u8;
    preview_mat_fn preview_from_materialized_u8;
    as_f64_fn materialized_logical_as_f64_u8;
    promote_inmem |x| f64::from(x);
    promote_spill |x| f64::from(x);
}

define_int_materialize! {
    u16;
    backing LogicalU16Backing;
    le_mod u16_le;
    scatter chunk_decode::scatter_chunk_into_plan_u16;
    scatter_seq scatter_fill_sequential_u16;
    scatter_ty ScatterU16Fn;
    scatter_par parallel::materialize_scatter_fill_parallel_u16;
    core_fn materialize_read_plan_u16_le_core;
    read_fn materialize_read_plan_u16_le;
    spill_fn spill_read_plan_u16_le;
    type_label "u16";
    into_vec_fn materialize_into_vec_u16;
    spill_file_fn preview_from_spill_file_u16;
    preview_mat_fn preview_from_materialized_u16;
    as_f64_fn materialized_logical_as_f64_u16;
    promote_inmem |x| f64::from(x);
    promote_spill |x| f64::from(x);
}

define_int_materialize! {
    u32;
    backing LogicalU32Backing;
    le_mod u32_le;
    scatter chunk_decode::scatter_chunk_into_plan_u32;
    scatter_seq scatter_fill_sequential_u32;
    scatter_ty ScatterU32Fn;
    scatter_par parallel::materialize_scatter_fill_parallel_u32;
    core_fn materialize_read_plan_u32_le_core;
    read_fn materialize_read_plan_u32_le;
    spill_fn spill_read_plan_u32_le;
    type_label "u32";
    into_vec_fn materialize_into_vec_u32;
    spill_file_fn preview_from_spill_file_u32;
    preview_mat_fn preview_from_materialized_u32;
    as_f64_fn materialized_logical_as_f64_u32;
    promote_inmem |x| f64::from(x);
    promote_spill |x| f64::from(x);
}

define_int_materialize! {
    u64;
    backing LogicalU64Backing;
    le_mod u64_le;
    scatter chunk_decode::scatter_chunk_into_plan_u64;
    scatter_seq scatter_fill_sequential_u64;
    scatter_ty ScatterU64Fn;
    scatter_par parallel::materialize_scatter_fill_parallel_u64;
    core_fn materialize_read_plan_u64_le_core;
    read_fn materialize_read_plan_u64_le;
    spill_fn spill_read_plan_u64_le;
    type_label "u64";
    into_vec_fn materialize_into_vec_u64;
    spill_file_fn preview_from_spill_file_u64;
    preview_mat_fn preview_from_materialized_u64;
    as_f64_fn materialized_logical_as_f64_u64;
    promote_inmem |x| x as f64;
    promote_spill |x| x as f64;
}

define_int_materialize! {
    i16;
    backing LogicalI16Backing;
    le_mod i16_le;
    scatter chunk_decode::scatter_chunk_into_plan_i16;
    scatter_seq scatter_fill_sequential_i16;
    scatter_ty ScatterI16Fn;
    scatter_par parallel::materialize_scatter_fill_parallel_i16;
    core_fn materialize_read_plan_i16_le_core;
    read_fn materialize_read_plan_i16_le;
    spill_fn spill_read_plan_i16_le;
    type_label "i16";
    into_vec_fn materialize_into_vec_i16;
    spill_file_fn preview_from_spill_file_i16;
    preview_mat_fn preview_from_materialized_i16;
    as_f64_fn materialized_logical_as_f64_i16;
    promote_inmem |x| f64::from(x);
    promote_spill |x| f64::from(x);
}

/// Spill a full logical integer selection to `path` (dispatches by `dtype`).
///
/// # Errors
///
/// Returns [`TetError::Validation`] when `dtype` is not an integer wire type, or on spill I/O failure.
pub fn spill_read_plan_int_le(
    mmap: &[u8],
    plan: &ReadPlan,
    path: &Path,
    dtype: ElementDtype,
) -> Result<u64, TetError> {
    match dtype {
        ElementDtype::I32 => spill_read_plan_i32_le(mmap, plan, path),
        ElementDtype::I64 => spill_read_plan_i64_le(mmap, plan, path),
        ElementDtype::U8 => spill_read_plan_u8_le(mmap, plan, path),
        ElementDtype::U16 => spill_read_plan_u16_le(mmap, plan, path),
        ElementDtype::I16 => spill_read_plan_i16_le(mmap, plan, path),
        ElementDtype::U32 => spill_read_plan_u32_le(mmap, plan, path),
        ElementDtype::U64 => spill_read_plan_u64_le(mmap, plan, path),
        _ => Err(TetError::Validation(
            "spill_read_plan_int_le requires an integer dtype (i32/i64/u8/u16/i16/u32/u64)".into(),
        )),
    }
}