tetration 0.1.8

Chunked tensor .tet format, tet CLI, and optional C ABI (tetration-ffi)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130

//! Footer `metadata` JSON roundtrip.

use super::fixture::write_multichunk_2x3_tiles;
use crate::catalog::{
    FooterBlobV1, TetMetadataV1, append_convert_history, read_footer_blob, read_tet_summary_v1,
    write_footer_blob,
};
use crate::layout::mmap_file_read;

#[test]
fn convert_history_preserves_existing_metadata() {
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("meta_convert.tet");
    write_multichunk_2x3_tiles(&path, "a");

    let meta = TetMetadataV1 {
        file: None,
        datasets: [(
            "a".to_owned(),
            crate::catalog::DatasetMetadataV1 {
                attrs: [("units".to_owned(), "K".to_owned())].into_iter().collect(),
                dim_names: Some(vec!["y".to_owned(), "x".to_owned()]),
                coords: None,
            },
        )]
        .into_iter()
        .collect(),
    };
    write_footer_blob(
        &path,
        &FooterBlobV1 {
            history: Vec::new(),
            metadata: Some(meta),
            metadata_ref: None,
        },
    )
    .unwrap();

    append_convert_history(&path, "h5").unwrap();

    let mmap = mmap_file_read(&path).unwrap();
    let summary = read_tet_summary_v1(&mmap).unwrap();
    assert_eq!(summary.history.len(), 1);
    assert_eq!(summary.history[0].op, "convert");
    assert_eq!(
        summary.metadata.datasets["a"]
            .attrs
            .get("units")
            .map(String::as_str),
        Some("K")
    );

    let blob = read_footer_blob(&mmap, summary.superblock.flags).unwrap();
    assert!(blob.metadata.is_some());
    assert_eq!(blob.history.len(), 1);
}

#[test]
fn footer_reads_legacy_history_triples() {
    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("legacy_hist.tet");
    write_multichunk_2x3_tiles(&path, "a");

    let legacy = br#"{"history":[["convert","h5","1700000000"]]}"#;
    crate::catalog::rewrite_footer_bytes_for_test(&path, legacy.to_vec()).unwrap();

    let mmap = crate::layout::mmap_file_read(&path).unwrap();
    let summary = read_tet_summary_v1(&mmap).unwrap();
    assert_eq!(summary.history.len(), 1);
    assert_eq!(summary.history[0].op, "convert");
    assert_eq!(summary.history[0].source, "h5");
    assert_eq!(summary.history[0].at, "1700000000");
}

#[test]
fn footer_spills_oversized_metadata() {
    use std::collections::BTreeMap;

    let dir = tempfile::tempdir().unwrap();
    let path = dir.path().join("spill_meta.tet");
    write_multichunk_2x3_tiles(&path, "a");

    let label = "x".repeat(1024);
    let mut coords = BTreeMap::new();
    for ax in 0..8 {
        coords.insert(
            format!("axis_{ax}"),
            crate::catalog::CoordAxisV1 {
                labels: vec![label.clone(); 64],
            },
        );
    }
    let meta = TetMetadataV1 {
        file: None,
        datasets: [(
            "a".to_owned(),
            crate::catalog::DatasetMetadataV1 {
                attrs: BTreeMap::new(),
                dim_names: None,
                coords: Some(coords),
            },
        )]
        .into_iter()
        .collect(),
    };
    write_footer_blob(
        &path,
        &FooterBlobV1 {
            history: Vec::new(),
            metadata: Some(meta.clone()),
            metadata_ref: None,
        },
    )
    .unwrap();

    let mmap = mmap_file_read(&path).unwrap();
    let summary = read_tet_summary_v1(&mmap).unwrap();
    assert_eq!(
        summary.metadata.datasets["a"]
            .coords
            .as_ref()
            .map(|c| c.len()),
        Some(8)
    );

    let raw = std::fs::read(&path).unwrap();
    let blob = read_footer_blob(&raw, summary.superblock.flags).unwrap();
    assert!(blob.metadata.is_some());
    assert!(raw.windows(12).any(|w| w == b"metadata_ref"));
}