hdf5_core/
lib.rs

1//! Shared HDF5 format model used by reader and writer crates.
2
3use std::fmt;
4
5/// Errors produced by shared HDF5 model helpers.
6#[derive(Debug, thiserror::Error)]
7pub enum Error {
8    #[error("invalid data: {0}")]
9    InvalidData(String),
10}
11
12pub type Result<T> = std::result::Result<T, Error>;
13
14/// Jenkins lookup3 `hashlittle2` used by HDF5 metadata checksums.
15///
16/// HDF5 uses this checksum for superblock v2/v3, object header v2,
17/// B-tree v2, extensible-array, fixed-array, fractal-heap, and shared-message
18/// metadata blocks. This is a direct translation of Bob Jenkins'
19/// lookup3.c `hashlittle2` with both init values set to zero.
20pub fn jenkins_lookup3(data: &[u8]) -> u32 {
21    let len = data.len();
22    let mut a: u32 = 0xdeadbeef_u32.wrapping_add(len as u32);
23    let mut b: u32 = a;
24    let mut c: u32 = a;
25    let mut offset = 0;
26
27    while offset + 12 < len {
28        a = a.wrapping_add(u32::from_le_bytes([
29            data[offset],
30            data[offset + 1],
31            data[offset + 2],
32            data[offset + 3],
33        ]));
34        b = b.wrapping_add(u32::from_le_bytes([
35            data[offset + 4],
36            data[offset + 5],
37            data[offset + 6],
38            data[offset + 7],
39        ]));
40        c = c.wrapping_add(u32::from_le_bytes([
41            data[offset + 8],
42            data[offset + 9],
43            data[offset + 10],
44            data[offset + 11],
45        ]));
46        jenkins_mix(&mut a, &mut b, &mut c);
47        offset += 12;
48    }
49
50    let remaining = len - offset;
51    if remaining > 0 {
52        let mut tail_a: u32 = 0;
53        let mut tail_b: u32 = 0;
54        let mut tail_c: u32 = 0;
55
56        for i in 0..remaining.min(4) {
57            tail_a |= (data[offset + i] as u32) << (i * 8);
58        }
59        if remaining > 4 {
60            for i in 4..remaining.min(8) {
61                tail_b |= (data[offset + i] as u32) << ((i - 4) * 8);
62            }
63        }
64        if remaining > 8 {
65            for i in 8..remaining {
66                tail_c |= (data[offset + i] as u32) << ((i - 8) * 8);
67            }
68        }
69
70        a = a.wrapping_add(tail_a);
71        b = b.wrapping_add(tail_b);
72        c = c.wrapping_add(tail_c);
73        jenkins_final_mix(&mut a, &mut b, &mut c);
74    }
75
76    c
77}
78
79/// Fletcher-32 checksum used by the HDF5 filter pipeline.
80///
81/// Matches HDF5's `H5_checksum_fletcher32`: reads 16-bit big-endian words,
82/// accumulates in batches of 360, and reduces with `(x & 0xffff) + (x >> 16)`.
83pub fn fletcher32(data: &[u8]) -> u32 {
84    let mut sum1: u32 = 0;
85    let mut sum2: u32 = 0;
86    let total_words = data.len() / 2;
87
88    let mut offset = 0usize;
89    let mut remaining = total_words;
90
91    while remaining > 0 {
92        let batch = remaining.min(360);
93        remaining -= batch;
94
95        for _ in 0..batch {
96            let word = ((data[offset] as u32) << 8) | (data[offset + 1] as u32);
97            sum1 += word;
98            sum2 += sum1;
99            offset += 2;
100        }
101
102        sum1 = (sum1 & 0xffff) + (sum1 >> 16);
103        sum2 = (sum2 & 0xffff) + (sum2 >> 16);
104    }
105
106    sum1 = (sum1 & 0xffff) + (sum1 >> 16);
107    sum2 = (sum2 & 0xffff) + (sum2 >> 16);
108
109    (sum2 << 16) | sum1
110}
111
112#[inline]
113fn jenkins_mix(a: &mut u32, b: &mut u32, c: &mut u32) {
114    *a = a.wrapping_sub(*c);
115    *a ^= c.rotate_left(4);
116    *c = c.wrapping_add(*b);
117    *b = b.wrapping_sub(*a);
118    *b ^= a.rotate_left(6);
119    *a = a.wrapping_add(*c);
120    *c = c.wrapping_sub(*b);
121    *c ^= b.rotate_left(8);
122    *b = b.wrapping_add(*a);
123    *a = a.wrapping_sub(*c);
124    *a ^= c.rotate_left(16);
125    *c = c.wrapping_add(*b);
126    *b = b.wrapping_sub(*a);
127    *b ^= a.rotate_left(19);
128    *a = a.wrapping_add(*c);
129    *c = c.wrapping_sub(*b);
130    *c ^= b.rotate_left(4);
131    *b = b.wrapping_add(*a);
132}
133
134#[inline]
135fn jenkins_final_mix(a: &mut u32, b: &mut u32, c: &mut u32) {
136    *c ^= *b;
137    *c = c.wrapping_sub(b.rotate_left(14));
138    *a ^= *c;
139    *a = a.wrapping_sub(c.rotate_left(11));
140    *b ^= *a;
141    *b = b.wrapping_sub(a.rotate_left(25));
142    *c ^= *b;
143    *c = c.wrapping_sub(b.rotate_left(16));
144    *a ^= *c;
145    *a = a.wrapping_sub(c.rotate_left(4));
146    *b ^= *a;
147    *b = b.wrapping_sub(a.rotate_left(14));
148    *c ^= *b;
149    *c = c.wrapping_sub(b.rotate_left(24));
150}
151
152/// Byte order for numeric data.
153#[derive(Debug, Clone, Copy, PartialEq, Eq)]
154pub enum ByteOrder {
155    LittleEndian,
156    BigEndian,
157}
158
159impl fmt::Display for ByteOrder {
160    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
161        match self {
162            ByteOrder::LittleEndian => write!(f, "little-endian"),
163            ByteOrder::BigEndian => write!(f, "big-endian"),
164        }
165    }
166}
167
168/// How a string's length is determined.
169#[derive(Debug, Clone, Copy, PartialEq, Eq)]
170pub enum StringSize {
171    /// Fixed-length, padded to `n` bytes.
172    Fixed(u32),
173    /// Variable-length.
174    Variable,
175}
176
177/// String character encoding.
178#[derive(Debug, Clone, Copy, PartialEq, Eq)]
179pub enum StringEncoding {
180    Ascii,
181    Utf8,
182}
183
184/// String padding type.
185#[derive(Debug, Clone, Copy, PartialEq, Eq)]
186pub enum StringPadding {
187    NullTerminate,
188    NullPad,
189    SpacePad,
190}
191
192/// HDF5 variable-length datatype flavor.
193#[derive(Debug, Clone, Copy, PartialEq, Eq)]
194pub enum VarLenKind {
195    /// Variable-length sequence of values.
196    Sequence,
197    /// Variable-length string.
198    String,
199    /// Unknown HDF5 vlen kind; retained for metadata fidelity.
200    Unknown(u8),
201}
202
203/// A field within a compound datatype.
204#[derive(Debug, Clone, PartialEq, Eq)]
205pub struct CompoundField {
206    pub name: String,
207    pub byte_offset: u32,
208    pub datatype: Datatype,
209}
210
211/// A member of an enumeration.
212#[derive(Debug, Clone, PartialEq, Eq)]
213pub struct EnumMember {
214    pub name: String,
215    pub value: Vec<u8>,
216}
217
218/// HDF5 reference type.
219#[derive(Debug, Clone, Copy, PartialEq, Eq)]
220pub enum ReferenceType {
221    /// Object reference.
222    Object,
223    /// Dataset region reference.
224    DatasetRegion,
225}
226
227/// Describes the element type of a dataset or attribute.
228#[derive(Debug, Clone, PartialEq, Eq)]
229pub enum Datatype {
230    FixedPoint {
231        size: u8,
232        signed: bool,
233        byte_order: ByteOrder,
234    },
235    FloatingPoint {
236        size: u8,
237        byte_order: ByteOrder,
238    },
239    String {
240        size: StringSize,
241        encoding: StringEncoding,
242        padding: StringPadding,
243    },
244    Compound {
245        size: u32,
246        fields: Vec<CompoundField>,
247    },
248    Array {
249        base: Box<Datatype>,
250        dims: Vec<u64>,
251    },
252    Enum {
253        base: Box<Datatype>,
254        members: Vec<EnumMember>,
255    },
256    VarLen {
257        base: Box<Datatype>,
258        kind: VarLenKind,
259        encoding: StringEncoding,
260        padding: StringPadding,
261    },
262    Opaque {
263        size: u32,
264        tag: String,
265    },
266    Reference {
267        ref_type: ReferenceType,
268        size: u8,
269    },
270    Bitfield {
271        size: u8,
272        byte_order: ByteOrder,
273    },
274}
275
276/// Datatype message payload plus element size from the message header.
277#[derive(Debug, Clone, PartialEq, Eq)]
278pub struct DatatypeMessage {
279    pub datatype: Datatype,
280    pub size: u32,
281}
282
283/// Unlimited dimension sentinel value.
284pub const UNLIMITED: u64 = u64::MAX;
285
286/// HDF5 dataspace kind.
287#[derive(Debug, Clone, Copy, PartialEq, Eq)]
288pub enum DataspaceType {
289    Null,
290    Scalar,
291    Simple,
292}
293
294/// Parsed dataspace message.
295#[derive(Debug, Clone, PartialEq, Eq)]
296pub struct DataspaceMessage {
297    pub rank: u8,
298    pub dims: Vec<u64>,
299    pub max_dims: Option<Vec<u64>>,
300    pub dataspace_type: DataspaceType,
301}
302
303impl DataspaceMessage {
304    /// Total number of elements in the dataspace.
305    pub fn num_elements(&self) -> Result<u64> {
306        if self.dims.is_empty() {
307            return Ok(match self.dataspace_type {
308                DataspaceType::Scalar => 1,
309                _ => 0,
310            });
311        }
312        self.dims.iter().try_fold(1u64, |acc, &dim| {
313            acc.checked_mul(dim).ok_or_else(|| {
314                Error::InvalidData("dataspace element count overflows u64".to_string())
315            })
316        })
317    }
318}
319
320/// Chunk indexing method for v4/v5 layout messages.
321#[derive(Debug, Clone, PartialEq, Eq)]
322pub enum ChunkIndexing {
323    SingleChunk {
324        filtered_size: u64,
325        filters: u32,
326    },
327    Implicit,
328    FixedArray {
329        page_bits: u8,
330        chunk_size_len: u8,
331    },
332    ExtensibleArray {
333        max_bits: u8,
334        index_bits: u8,
335        min_pointers: u8,
336        min_elements: u8,
337        chunk_size_len: u8,
338    },
339    BTreeV2,
340}
341
342/// Raw data storage layout for a dataset.
343#[derive(Debug, Clone, PartialEq, Eq)]
344pub enum DataLayout {
345    Compact {
346        data: Vec<u8>,
347    },
348    Contiguous {
349        address: u64,
350        size: u64,
351    },
352    Chunked {
353        address: u64,
354        dims: Vec<u32>,
355        element_size: u32,
356        chunk_indexing: Option<ChunkIndexing>,
357    },
358}
359
360/// Parsed data layout message.
361#[derive(Debug, Clone, PartialEq, Eq)]
362pub struct DataLayoutMessage {
363    pub layout: DataLayout,
364}
365
366/// When to write the fill value.
367#[derive(Debug, Clone, Copy, PartialEq, Eq)]
368pub enum FillTime {
369    IfSet,
370    Always,
371    Never,
372}
373
374/// Parsed fill value message.
375#[derive(Debug, Clone, PartialEq, Eq)]
376pub struct FillValueMessage {
377    pub defined: bool,
378    pub fill_time: FillTime,
379    pub value: Option<Vec<u8>>,
380}
381
382/// Standard HDF5 filter IDs.
383pub const FILTER_DEFLATE: u16 = 1;
384pub const FILTER_SHUFFLE: u16 = 2;
385pub const FILTER_FLETCHER32: u16 = 3;
386pub const FILTER_SZIP: u16 = 4;
387pub const FILTER_NBIT: u16 = 5;
388pub const FILTER_SCALEOFFSET: u16 = 6;
389pub const FILTER_LZ4: u16 = 32004;
390
391/// A single filter in a filter pipeline.
392#[derive(Debug, Clone, PartialEq, Eq)]
393pub struct FilterDescription {
394    pub id: u16,
395    pub name: Option<String>,
396    pub client_data: Vec<u32>,
397}
398
399/// Parsed filter pipeline message.
400#[derive(Debug, Clone, PartialEq, Eq)]
401pub struct FilterPipelineMessage {
402    pub filters: Vec<FilterDescription>,
403}
404
405/// A single external raw data file slot.
406#[derive(Debug, Clone, PartialEq, Eq)]
407pub struct ExternalFileSlot {
408    pub name_offset: u64,
409    pub offset: u64,
410    pub size: u64,
411}
412
413/// Parsed external raw data files message.
414#[derive(Debug, Clone, PartialEq, Eq)]
415pub struct ExternalFilesMessage {
416    pub heap_address: u64,
417    pub slots: Vec<ExternalFileSlot>,
418}
419
420/// Where an HDF5 link points.
421#[derive(Debug, Clone, PartialEq, Eq)]
422pub enum LinkTarget {
423    Hard { address: u64 },
424    Soft { path: String },
425    External { filename: String, path: String },
426}
427
428/// Parsed link message.
429#[derive(Debug, Clone, PartialEq, Eq)]
430pub struct LinkMessage {
431    pub name: String,
432    pub target: LinkTarget,
433    pub creation_order: Option<u64>,
434}
435
436#[cfg(test)]
437mod tests {
438    use super::*;
439
440    #[test]
441    fn jenkins_lookup3_is_deterministic() {
442        assert_eq!(jenkins_lookup3(b"hello"), jenkins_lookup3(b"hello"));
443        assert_ne!(jenkins_lookup3(b"hello"), jenkins_lookup3(b"world"));
444    }
445
446    #[test]
447    fn jenkins_lookup3_handles_twelve_byte_boundary_like_lookup3() {
448        let twelve = [0x5au8; 12];
449        let thirteen = [0x5au8; 13];
450
451        assert_ne!(jenkins_lookup3(&twelve), jenkins_lookup3(&thirteen));
452        assert_eq!(jenkins_lookup3(&twelve), jenkins_lookup3(&twelve));
453    }
454
455    #[test]
456    fn fletcher32_is_deterministic() {
457        let data = [0x01, 0x02, 0x03, 0x04];
458
459        assert_eq!(fletcher32(&data), fletcher32(&data));
460    }
461
462    #[test]
463    fn fletcher32_matches_known_reference() {
464        let data = [0x00, 0x01, 0x00, 0x02];
465
466        assert_eq!(fletcher32(&data), 0x0004_0003);
467    }
468}
hdf5_core/lib.rs

hdf5_core/
lib.rs
