hdf5_core/
lib.rs

1//! Shared HDF5 format model used by reader and writer crates.
2
3use std::fmt;
4
5/// Errors produced by shared HDF5 model helpers.
6#[derive(Debug, thiserror::Error)]
7pub enum Error {
8    #[error("invalid data: {0}")]
9    InvalidData(String),
10}
11
12pub type Result<T> = std::result::Result<T, Error>;
13
14/// Jenkins lookup3 `hashlittle2` used by HDF5 metadata checksums.
15///
16/// HDF5 uses this checksum for superblock v2/v3, object header v2,
17/// B-tree v2, extensible-array, fixed-array, fractal-heap, and shared-message
18/// metadata blocks. This is a direct translation of Bob Jenkins'
19/// lookup3.c `hashlittle2` with both init values set to zero.
20pub fn jenkins_lookup3(data: &[u8]) -> u32 {
21    let len = data.len();
22    let mut a: u32 = 0xdeadbeef_u32.wrapping_add(len as u32);
23    let mut b: u32 = a;
24    let mut c: u32 = a;
25    let mut offset = 0;
26
27    while offset + 12 < len {
28        a = a.wrapping_add(u32::from_le_bytes([
29            data[offset],
30            data[offset + 1],
31            data[offset + 2],
32            data[offset + 3],
33        ]));
34        b = b.wrapping_add(u32::from_le_bytes([
35            data[offset + 4],
36            data[offset + 5],
37            data[offset + 6],
38            data[offset + 7],
39        ]));
40        c = c.wrapping_add(u32::from_le_bytes([
41            data[offset + 8],
42            data[offset + 9],
43            data[offset + 10],
44            data[offset + 11],
45        ]));
46        jenkins_mix(&mut a, &mut b, &mut c);
47        offset += 12;
48    }
49
50    let remaining = len - offset;
51    if remaining > 0 {
52        let mut tail_a: u32 = 0;
53        let mut tail_b: u32 = 0;
54        let mut tail_c: u32 = 0;
55
56        for i in 0..remaining.min(4) {
57            tail_a |= (data[offset + i] as u32) << (i * 8);
58        }
59        if remaining > 4 {
60            for i in 4..remaining.min(8) {
61                tail_b |= (data[offset + i] as u32) << ((i - 4) * 8);
62            }
63        }
64        if remaining > 8 {
65            for i in 8..remaining {
66                tail_c |= (data[offset + i] as u32) << ((i - 8) * 8);
67            }
68        }
69
70        a = a.wrapping_add(tail_a);
71        b = b.wrapping_add(tail_b);
72        c = c.wrapping_add(tail_c);
73        jenkins_final_mix(&mut a, &mut b, &mut c);
74    }
75
76    c
77}
78
79/// Fletcher-32 checksum used by the HDF5 filter pipeline.
80///
81/// Matches HDF5's `H5_checksum_fletcher32`: reads 16-bit big-endian words,
82/// accumulates in batches of 360, and reduces with `(x & 0xffff) + (x >> 16)`.
83pub fn fletcher32(data: &[u8]) -> u32 {
84    let mut sum1: u32 = 0;
85    let mut sum2: u32 = 0;
86    let total_words = data.len() / 2;
87
88    let mut offset = 0usize;
89    let mut remaining = total_words;
90
91    while remaining > 0 {
92        let batch = remaining.min(360);
93        remaining -= batch;
94
95        for _ in 0..batch {
96            let word = ((data[offset] as u32) << 8) | (data[offset + 1] as u32);
97            sum1 += word;
98            sum2 += sum1;
99            offset += 2;
100        }
101
102        sum1 = (sum1 & 0xffff) + (sum1 >> 16);
103        sum2 = (sum2 & 0xffff) + (sum2 >> 16);
104    }
105
106    // A trailing odd byte contributes as the high half of a final word.
107    if data.len() % 2 == 1 {
108        sum1 += (data[data.len() - 1] as u32) << 8;
109        sum2 += sum1;
110        sum1 = (sum1 & 0xffff) + (sum1 >> 16);
111        sum2 = (sum2 & 0xffff) + (sum2 >> 16);
112    }
113
114    sum1 = (sum1 & 0xffff) + (sum1 >> 16);
115    sum2 = (sum2 & 0xffff) + (sum2 >> 16);
116
117    (sum2 << 16) | sum1
118}
119
120#[inline]
121fn jenkins_mix(a: &mut u32, b: &mut u32, c: &mut u32) {
122    *a = a.wrapping_sub(*c);
123    *a ^= c.rotate_left(4);
124    *c = c.wrapping_add(*b);
125    *b = b.wrapping_sub(*a);
126    *b ^= a.rotate_left(6);
127    *a = a.wrapping_add(*c);
128    *c = c.wrapping_sub(*b);
129    *c ^= b.rotate_left(8);
130    *b = b.wrapping_add(*a);
131    *a = a.wrapping_sub(*c);
132    *a ^= c.rotate_left(16);
133    *c = c.wrapping_add(*b);
134    *b = b.wrapping_sub(*a);
135    *b ^= a.rotate_left(19);
136    *a = a.wrapping_add(*c);
137    *c = c.wrapping_sub(*b);
138    *c ^= b.rotate_left(4);
139    *b = b.wrapping_add(*a);
140}
141
142#[inline]
143fn jenkins_final_mix(a: &mut u32, b: &mut u32, c: &mut u32) {
144    *c ^= *b;
145    *c = c.wrapping_sub(b.rotate_left(14));
146    *a ^= *c;
147    *a = a.wrapping_sub(c.rotate_left(11));
148    *b ^= *a;
149    *b = b.wrapping_sub(a.rotate_left(25));
150    *c ^= *b;
151    *c = c.wrapping_sub(b.rotate_left(16));
152    *a ^= *c;
153    *a = a.wrapping_sub(c.rotate_left(4));
154    *b ^= *a;
155    *b = b.wrapping_sub(a.rotate_left(14));
156    *c ^= *b;
157    *c = c.wrapping_sub(b.rotate_left(24));
158}
159
160/// Byte order for numeric data.
161#[derive(Debug, Clone, Copy, PartialEq, Eq)]
162pub enum ByteOrder {
163    LittleEndian,
164    BigEndian,
165}
166
167impl fmt::Display for ByteOrder {
168    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
169        match self {
170            ByteOrder::LittleEndian => write!(f, "little-endian"),
171            ByteOrder::BigEndian => write!(f, "big-endian"),
172        }
173    }
174}
175
176/// How a string's length is determined.
177#[derive(Debug, Clone, Copy, PartialEq, Eq)]
178pub enum StringSize {
179    /// Fixed-length, padded to `n` bytes.
180    Fixed(u32),
181    /// Variable-length.
182    Variable,
183}
184
185/// String character encoding.
186#[derive(Debug, Clone, Copy, PartialEq, Eq)]
187pub enum StringEncoding {
188    Ascii,
189    Utf8,
190}
191
192/// String padding type.
193#[derive(Debug, Clone, Copy, PartialEq, Eq)]
194pub enum StringPadding {
195    NullTerminate,
196    NullPad,
197    SpacePad,
198}
199
200/// HDF5 variable-length datatype flavor.
201#[derive(Debug, Clone, Copy, PartialEq, Eq)]
202pub enum VarLenKind {
203    /// Variable-length sequence of values.
204    Sequence,
205    /// Variable-length string.
206    String,
207    /// Unknown HDF5 vlen kind; retained for metadata fidelity.
208    Unknown(u8),
209}
210
211/// A field within a compound datatype.
212#[derive(Debug, Clone, PartialEq, Eq)]
213pub struct CompoundField {
214    pub name: String,
215    pub byte_offset: u32,
216    pub datatype: Datatype,
217}
218
219/// A member of an enumeration.
220#[derive(Debug, Clone, PartialEq, Eq)]
221pub struct EnumMember {
222    pub name: String,
223    pub value: Vec<u8>,
224}
225
226/// HDF5 reference type.
227#[derive(Debug, Clone, Copy, PartialEq, Eq)]
228pub enum ReferenceType {
229    /// Object reference.
230    Object,
231    /// Dataset region reference.
232    DatasetRegion,
233}
234
235/// Describes the element type of a dataset or attribute.
236#[derive(Debug, Clone, PartialEq, Eq)]
237pub enum Datatype {
238    FixedPoint {
239        size: u8,
240        signed: bool,
241        byte_order: ByteOrder,
242    },
243    FloatingPoint {
244        size: u8,
245        byte_order: ByteOrder,
246    },
247    String {
248        size: StringSize,
249        encoding: StringEncoding,
250        padding: StringPadding,
251    },
252    Compound {
253        size: u32,
254        fields: Vec<CompoundField>,
255    },
256    Array {
257        base: Box<Datatype>,
258        dims: Vec<u64>,
259    },
260    Enum {
261        base: Box<Datatype>,
262        members: Vec<EnumMember>,
263    },
264    VarLen {
265        base: Box<Datatype>,
266        kind: VarLenKind,
267        encoding: StringEncoding,
268        padding: StringPadding,
269    },
270    Opaque {
271        size: u32,
272        tag: String,
273    },
274    Reference {
275        ref_type: ReferenceType,
276        size: u8,
277    },
278    Bitfield {
279        size: u8,
280        byte_order: ByteOrder,
281    },
282}
283
284/// Datatype message payload plus element size from the message header.
285#[derive(Debug, Clone, PartialEq, Eq)]
286pub struct DatatypeMessage {
287    pub datatype: Datatype,
288    pub size: u32,
289}
290
291/// Unlimited dimension sentinel value.
292pub const UNLIMITED: u64 = u64::MAX;
293
294/// HDF5 dataspace kind.
295#[derive(Debug, Clone, Copy, PartialEq, Eq)]
296pub enum DataspaceType {
297    Null,
298    Scalar,
299    Simple,
300}
301
302/// Parsed dataspace message.
303#[derive(Debug, Clone, PartialEq, Eq)]
304pub struct DataspaceMessage {
305    pub rank: u8,
306    pub dims: Vec<u64>,
307    pub max_dims: Option<Vec<u64>>,
308    pub dataspace_type: DataspaceType,
309}
310
311impl DataspaceMessage {
312    /// Total number of elements in the dataspace.
313    pub fn num_elements(&self) -> Result<u64> {
314        if self.dims.is_empty() {
315            return Ok(match self.dataspace_type {
316                DataspaceType::Scalar => 1,
317                _ => 0,
318            });
319        }
320        self.dims.iter().try_fold(1u64, |acc, &dim| {
321            acc.checked_mul(dim).ok_or_else(|| {
322                Error::InvalidData("dataspace element count overflows u64".to_string())
323            })
324        })
325    }
326}
327
328/// Chunk indexing method for v4/v5 layout messages.
329#[derive(Debug, Clone, PartialEq, Eq)]
330pub enum ChunkIndexing {
331    SingleChunk {
332        filtered_size: u64,
333        filters: u32,
334    },
335    Implicit,
336    FixedArray {
337        page_bits: u8,
338        chunk_size_len: u8,
339    },
340    ExtensibleArray {
341        max_bits: u8,
342        index_bits: u8,
343        min_pointers: u8,
344        min_elements: u8,
345        chunk_size_len: u8,
346    },
347    BTreeV2,
348}
349
350/// Raw data storage layout for a dataset.
351#[derive(Debug, Clone, PartialEq, Eq)]
352pub enum DataLayout {
353    Compact {
354        data: Vec<u8>,
355    },
356    Contiguous {
357        address: u64,
358        size: u64,
359    },
360    Chunked {
361        address: u64,
362        dims: Vec<u32>,
363        element_size: u32,
364        chunk_indexing: Option<ChunkIndexing>,
365    },
366}
367
368/// Parsed data layout message.
369#[derive(Debug, Clone, PartialEq, Eq)]
370pub struct DataLayoutMessage {
371    pub layout: DataLayout,
372}
373
374/// When to write the fill value.
375#[derive(Debug, Clone, Copy, PartialEq, Eq)]
376pub enum FillTime {
377    IfSet,
378    Always,
379    Never,
380}
381
382/// Parsed fill value message.
383#[derive(Debug, Clone, PartialEq, Eq)]
384pub struct FillValueMessage {
385    pub defined: bool,
386    pub fill_time: FillTime,
387    pub value: Option<Vec<u8>>,
388}
389
390/// Standard HDF5 filter IDs.
391pub const FILTER_DEFLATE: u16 = 1;
392pub const FILTER_SHUFFLE: u16 = 2;
393pub const FILTER_FLETCHER32: u16 = 3;
394pub const FILTER_SZIP: u16 = 4;
395pub const FILTER_NBIT: u16 = 5;
396pub const FILTER_SCALEOFFSET: u16 = 6;
397pub const FILTER_LZ4: u16 = 32004;
398
399/// A single filter in a filter pipeline.
400#[derive(Debug, Clone, PartialEq, Eq)]
401pub struct FilterDescription {
402    pub id: u16,
403    pub name: Option<String>,
404    pub client_data: Vec<u32>,
405}
406
407/// Parsed filter pipeline message.
408#[derive(Debug, Clone, PartialEq, Eq)]
409pub struct FilterPipelineMessage {
410    pub filters: Vec<FilterDescription>,
411}
412
413/// A single external raw data file slot.
414#[derive(Debug, Clone, PartialEq, Eq)]
415pub struct ExternalFileSlot {
416    pub name_offset: u64,
417    pub offset: u64,
418    pub size: u64,
419}
420
421/// Parsed external raw data files message.
422#[derive(Debug, Clone, PartialEq, Eq)]
423pub struct ExternalFilesMessage {
424    pub heap_address: u64,
425    pub slots: Vec<ExternalFileSlot>,
426}
427
428/// Where an HDF5 link points.
429#[derive(Debug, Clone, PartialEq, Eq)]
430pub enum LinkTarget {
431    Hard { address: u64 },
432    Soft { path: String },
433    External { filename: String, path: String },
434}
435
436/// Parsed link message.
437#[derive(Debug, Clone, PartialEq, Eq)]
438pub struct LinkMessage {
439    pub name: String,
440    pub target: LinkTarget,
441    pub creation_order: Option<u64>,
442}
443
444#[cfg(test)]
445mod tests {
446    use super::*;
447
448    #[test]
449    fn jenkins_lookup3_is_deterministic() {
450        assert_eq!(jenkins_lookup3(b"hello"), jenkins_lookup3(b"hello"));
451        assert_ne!(jenkins_lookup3(b"hello"), jenkins_lookup3(b"world"));
452    }
453
454    #[test]
455    fn jenkins_lookup3_handles_twelve_byte_boundary_like_lookup3() {
456        let twelve = [0x5au8; 12];
457        let thirteen = [0x5au8; 13];
458
459        assert_ne!(jenkins_lookup3(&twelve), jenkins_lookup3(&thirteen));
460        assert_eq!(jenkins_lookup3(&twelve), jenkins_lookup3(&twelve));
461    }
462
463    #[test]
464    fn fletcher32_is_deterministic() {
465        let data = [0x01, 0x02, 0x03, 0x04];
466
467        assert_eq!(fletcher32(&data), fletcher32(&data));
468    }
469
470    #[test]
471    fn fletcher32_matches_known_reference() {
472        let data = [0x00, 0x01, 0x00, 0x02];
473
474        assert_eq!(fletcher32(&data), 0x0004_0003);
475    }
476
477    #[test]
478    fn fletcher32_handles_trailing_odd_byte_like_libhdf5() {
479        // H5_checksum_fletcher32 folds a trailing odd byte in as the high
480        // half of a final 16-bit word.
481        let data = [0x01, 0x02, 0x03];
482
483        assert_eq!(fletcher32(&data), 0x0504_0402);
484    }
485}
hdf5_core/lib.rs

hdf5_core/
lib.rs
