hdf5file/lowlevel/
level2.rs

1use crate::io::{ReadExt as _, SeekExt as _};
2use crate::{Error, ErrorKind, Result};
3use ndarray;
4use ndarray::ArrayD;
5use std;
6use std::convert::TryFrom;
7use std::io::{Read, Seek};
8
9/// Data object.
10#[derive(Debug)]
11pub enum DataObject {
12    /// Floating-point numbers.
13    Float(ArrayD<f64>),
14}
15
16// TODO: move level2a
17/// https://support.hdfgroup.org/HDF5/doc/H5.format.html#ObjectHeader
18#[derive(Debug, Clone)]
19pub struct ObjectHeader {
20    prefix: ObjectHeaderPrefix,
21}
22impl ObjectHeader {
23    pub fn from_reader<R: Read>(mut reader: R) -> Result<Self> {
24        let prefix = track!(ObjectHeaderPrefix::from_reader(&mut reader))?;
25        Ok(Self { prefix })
26    }
27
28    pub fn get_data_object<R: Read + Seek>(&self, mut reader: R) -> Result<DataObject> {
29        let bytes = track!(self.get_data_bytes(&mut reader))?;
30        let dimensions = track!(self.dimensions())?
31            .iter()
32            .map(|&d| d as usize)
33            .collect::<Vec<_>>();
34        let datatype = track!(self.datatype())?;
35
36        let count = dimensions.iter().cloned().product::<usize>();
37        let mut reader = &bytes[..];
38        match datatype {
39            DatatypeMessage::FloatingPoint(t) => {
40                let items = (0..count)
41                    .map(|i| track!(t.decode(&mut reader); i))
42                    .collect::<Result<Vec<_>>>()?;
43                track_assert_eq!(reader, b"", ErrorKind::InvalidFile);
44
45                let items = track!(ndarray::aview1(&items)
46                    .into_shape(dimensions)
47                    .map_err(Error::from))?;
48                Ok(DataObject::Float(items.to_owned()))
49            }
50            _ => track_panic!(ErrorKind::Unsupported),
51        }
52    }
53
54    fn dimensions(&self) -> Result<&[u64]> {
55        for m in &self.prefix.messages {
56            if let Message::Dataspace(m) = &m.message {
57                return Ok(&m.dimension_sizes);
58            }
59        }
60        track_panic!(ErrorKind::Other);
61    }
62
63    fn datatype(&self) -> Result<DatatypeMessage> {
64        for m in &self.prefix.messages {
65            if let Message::Datatype(m) = &m.message {
66                return Ok(m.clone());
67            }
68        }
69        track_panic!(ErrorKind::Other);
70    }
71
72    pub fn get_data_bytes<R: Read + Seek>(&self, mut reader: R) -> Result<Vec<u8>> {
73        for m in &self.prefix.messages {
74            if let Message::DataLayout(m) = &m.message {
75                let Layout::Contiguous { address, size } = m.layout;
76                track!(reader.seek_to(address))?;
77                return track!(reader.read_vec(size as usize));
78            }
79        }
80        track_panic!(ErrorKind::Other, "Not a data object");
81    }
82}
83
84#[derive(Debug, Clone)]
85pub struct ObjectHeaderPrefix {
86    messages: Vec<HeaderMessage>,
87    object_reference_count: u32,
88    object_header_size: u32,
89}
90impl ObjectHeaderPrefix {
91    pub fn from_reader<R: Read>(mut reader: R) -> Result<Self> {
92        let version = track!(reader.read_u8())?;
93        track_assert_eq!(version, 1, ErrorKind::InvalidFile);
94
95        let _reserved = track!(reader.read_u8())?;
96        track_assert_eq!(_reserved, 0, ErrorKind::InvalidFile);
97
98        let header_message_count = track!(reader.read_u16())?;
99        let object_reference_count = track!(reader.read_u32())?;
100        let object_header_size = track!(reader.read_u32())?;
101
102        // Header messages are aligned on 8-byte boundaries for version 1 object headers.
103        track!(reader.skip(4))?;
104
105        let mut reader = reader.take(u64::from(object_header_size));
106        let messages = (0..header_message_count)
107            .map(|_| track!(HeaderMessage::from_reader(&mut reader)))
108            .collect::<Result<_>>()?;
109        track_assert_eq!(reader.limit(), 0, ErrorKind::Other; object_header_size, messages);
110
111        Ok(Self {
112            messages,
113            object_reference_count,
114            object_header_size,
115        })
116    }
117}
118
119bitflags! {
120    struct HeaderMessageFlags: u8 {
121        const CONSTANT = 0b0000_0001;
122        const SHARED = 0b0000_0010;
123        const UNSHARABLE = 0b0000_0100;
124        const CANNOT_WRITE_IF_UNKNOWN = 0b0000_1000;
125        const SET_5_BIT_IF_UNKNOWN = 0b0001_0000;
126        const UNKNOWN_BUT_MODIFIED = 0b0010_0000;
127        const SHARABLE = 0b0100_0000;
128        const FAIL_IF_UNKNOWN = 0b0100_0000;
129    }
130}
131
132#[derive(Debug, Clone)]
133pub struct HeaderMessage {
134    flags: HeaderMessageFlags,
135    message: Message,
136}
137impl HeaderMessage {
138    pub fn from_reader<R: Read>(mut reader: R) -> Result<Self> {
139        let kind = track!(reader.read_u16())?;
140        let data_len = track!(reader.read_u16())?;
141        let flags = HeaderMessageFlags::from_bits_truncate(track!(reader.read_u8())?);
142        track!(reader.skip(3))?;
143        let mut reader = reader.take(u64::from(data_len));
144        let message = match kind {
145            0x00 => track!(NilMessage::from_reader(&mut reader)).map(Message::Nil)?,
146            0x01 => track!(DataspaceMessage::from_reader(&mut reader)).map(Message::Dataspace)?,
147            0x03 => track!(DatatypeMessage::from_reader(&mut reader)).map(Message::Datatype)?,
148            0x05 => track!(FillValueMessage::from_reader(&mut reader)).map(Message::FillValue)?,
149            0x08 => track!(DataLayoutMessage::from_reader(&mut reader)).map(Message::DataLayout)?,
150            0x11 => {
151                track!(SymbolTableMessage::from_reader(&mut reader)).map(Message::SymbolTable)?
152            }
153            0x12 => track!(ObjectModificationTimeMessage::from_reader(&mut reader))
154                .map(Message::ObjectModificationTime)?,
155            _ => track_panic!(ErrorKind::Unsupported, "Message type: {}", kind),
156        };
157        track_assert_eq!(reader.limit(), 0, ErrorKind::Other);
158
159        Ok(Self { flags, message })
160    }
161}
162
163/// type=0x00
164#[derive(Debug, Clone)]
165pub struct NilMessage {}
166impl NilMessage {
167    pub fn from_reader<R: Read>(mut reader: R) -> Result<Self> {
168        let _ = track!(reader.read_all())?;
169        Ok(Self {})
170    }
171}
172
173/// type=0x01
174#[derive(Debug, Clone)]
175pub struct DataspaceMessage {
176    dimension_sizes: Vec<u64>,
177    dimension_max_sizes: Option<Vec<u64>>,
178}
179impl DataspaceMessage {
180    pub fn from_reader<R: Read>(mut reader: R) -> Result<Self> {
181        let version = track!(reader.read_u8())?;
182        track_assert_eq!(version, 1, ErrorKind::Unsupported);
183
184        let dimensionality = track!(reader.read_u8())?;
185        let flags = track!(reader.read_u8())?; // TODO: consider flags
186        track!(reader.skip(5))?;
187
188        let dimension_sizes = (0..dimensionality)
189            .map(|_| track!(reader.read_u64()))
190            .collect::<Result<Vec<_>>>()?;
191
192        let dimension_max_sizes = if (flags & 0b0000_0001) != 0 {
193            Some(
194                (0..dimensionality)
195                    .map(|_| track!(reader.read_u64()))
196                    .collect::<Result<Vec<_>>>()?,
197            )
198        } else {
199            None
200        };
201
202        if (flags & 0b0000_0010) != 0 {
203            track_panic!(ErrorKind::Unsupported);
204        }
205
206        Ok(Self {
207            dimension_sizes,
208            dimension_max_sizes,
209        })
210    }
211}
212
213#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
214pub enum DatatypeClass {
215    FixedPoint,
216    FloatingPoint,
217    Time,
218    String,
219    BitField,
220    Opaque,
221    Compound,
222    Reference,
223    Enumerated,
224    VariableLength,
225    Array,
226}
227impl TryFrom<u8> for DatatypeClass {
228    type Error = Error;
229
230    fn try_from(f: u8) -> Result<Self> {
231        Ok(match f {
232            0 => DatatypeClass::FixedPoint,
233            1 => DatatypeClass::FloatingPoint,
234            2 => DatatypeClass::Time,
235            3 => DatatypeClass::String,
236            4 => DatatypeClass::BitField,
237            5 => DatatypeClass::Opaque,
238            6 => DatatypeClass::Compound,
239            7 => DatatypeClass::Reference,
240            8 => DatatypeClass::Enumerated,
241            9 => DatatypeClass::VariableLength,
242            10 => DatatypeClass::Array,
243            _ => track_panic!(ErrorKind::InvalidFile, "Unknown datatype class: {}", f),
244        })
245    }
246}
247
248#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
249pub enum MantissaNorm {
250    None,
251    AlwaysSet,
252    ImpliedToBeSet,
253}
254impl TryFrom<u8> for MantissaNorm {
255    type Error = Error;
256
257    fn try_from(f: u8) -> Result<Self> {
258        match f {
259            0 => Ok(MantissaNorm::None),
260            1 => Ok(MantissaNorm::AlwaysSet),
261            2 => Ok(MantissaNorm::ImpliedToBeSet),
262            3 => track_panic!(ErrorKind::InvalidFile, "Reserved value"),
263            _ => track_panic!(ErrorKind::InvalidInput),
264        }
265    }
266}
267
268#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
269pub enum Endian {
270    Little,
271    Big,
272    Vax,
273}
274impl TryFrom<u8> for Endian {
275    type Error = Error;
276
277    fn try_from(f: u8) -> Result<Self> {
278        match f {
279            0b0000_0000 => Ok(Endian::Little),
280            0b0000_0001 => Ok(Endian::Big),
281            0b0100_0000 => track_panic!(ErrorKind::InvalidFile, "Reserved endian bits"),
282            0b0100_0001 => Ok(Endian::Vax),
283            _ => track_panic!(ErrorKind::InvalidInput),
284        }
285    }
286}
287
288#[derive(Debug, Clone)]
289pub struct FloatingPointDatatype {
290    size: u32,
291
292    endian: Endian,
293    low_padding_bit: u8,
294    high_padding_bit: u8,
295    internal_padding_bit: u8,
296    mantissa_norm: MantissaNorm,
297    sign_location: u8,
298
299    bit_offset: u16,
300    bit_precision: u16,
301    exponent_location: u8,
302    exponent_size: u8,
303    mantissa_location: u8,
304    mantissa_size: u8,
305    exponent_bias: u32,
306}
307impl FloatingPointDatatype {
308    pub fn decode<R: Read>(&self, mut reader: R) -> Result<f64> {
309        track_assert_eq!(self.endian, Endian::Little, ErrorKind::Unsupported);
310        track_assert_eq!(self.low_padding_bit, 0, ErrorKind::Unsupported);
311        track_assert_eq!(self.high_padding_bit, 0, ErrorKind::Unsupported);
312        track_assert_eq!(self.internal_padding_bit, 0, ErrorKind::Unsupported);
313        track_assert_eq!(
314            self.mantissa_norm,
315            MantissaNorm::ImpliedToBeSet,
316            ErrorKind::Unsupported
317        );
318        track_assert_eq!(self.sign_location, 31, ErrorKind::Unsupported);
319
320        track_assert_eq!(self.bit_offset, 0, ErrorKind::Unsupported);
321        track_assert_eq!(self.bit_precision, 32, ErrorKind::Unsupported);
322        track_assert_eq!(self.exponent_location, 23, ErrorKind::Unsupported);
323        track_assert_eq!(self.exponent_size, 8, ErrorKind::Unsupported);
324        track_assert_eq!(self.mantissa_location, 0, ErrorKind::Unsupported);
325        track_assert_eq!(self.mantissa_size, 23, ErrorKind::Unsupported);
326        track_assert_eq!(self.exponent_bias, 127, ErrorKind::Unsupported);
327
328        track!(reader.read_f32()).map(f64::from)
329    }
330
331    pub fn from_reader<R: Read>(bit_field: u32, size: u32, mut reader: R) -> Result<Self> {
332        let bit_offset = track!(reader.read_u16())?;
333        let bit_precision = track!(reader.read_u16())?;
334        let exponent_location = track!(reader.read_u8())?;
335        let exponent_size = track!(reader.read_u8())?;
336        let mantissa_location = track!(reader.read_u8())?;
337        let mantissa_size = track!(reader.read_u8())?;
338        let exponent_bias = track!(reader.read_u32())?;
339        track!(reader.skip(4))?;
340
341        Ok(Self {
342            size,
343
344            endian: track!(Endian::try_from((bit_field & 0b0100_0001) as u8))?,
345            low_padding_bit: ((bit_field >> 1) & 1) as u8,
346            high_padding_bit: ((bit_field >> 2) & 1) as u8,
347            internal_padding_bit: ((bit_field >> 3) & 1) as u8,
348            mantissa_norm: track!(MantissaNorm::try_from(((bit_field >> 4) & 0b11) as u8))?,
349            sign_location: (bit_field >> 8) as u8,
350
351            bit_offset,
352            bit_precision,
353            exponent_location,
354            exponent_size,
355            mantissa_location,
356            mantissa_size,
357            exponent_bias,
358        })
359    }
360}
361
362#[derive(Debug, Clone)]
363pub struct FixedPointDatatype {
364    bit_field: u32,
365    size: u32,
366
367    bit_offset: u16,
368    bit_precision: u16,
369}
370impl FixedPointDatatype {
371    pub fn from_reader<R: Read>(bit_field: u32, size: u32, mut reader: R) -> Result<Self> {
372        let bit_offset = track!(reader.read_u16())?;
373        let bit_precision = track!(reader.read_u16())?;
374        track!(reader.skip(4))?;
375
376        Ok(Self {
377            bit_field,
378            size,
379
380            bit_offset,
381            bit_precision,
382        })
383    }
384}
385
386/// type=0x03
387#[derive(Debug, Clone)]
388pub enum DatatypeMessage {
389    FixedPoint(FixedPointDatatype),
390    FloatingPoint(FloatingPointDatatype),
391    // Time,
392    // String,
393    // BitField,
394    // Opaque,
395    // Compound,
396    // Reference,
397    // Enumerated,
398    // VariableLength,
399    // Array,
400}
401impl DatatypeMessage {
402    // pub fn decode<R: Read>(&self, reader: R) -> Result<DataItem> {
403    //     match self {
404    //         DatatypeMessage::FloatingPoint(t) => track!(t.decode(reader)).map(DataItem::Float),
405    //         _ => track_panic!(ErrorKind::Unsupported),
406    //     }
407    // }
408
409    pub fn from_reader<R: Read>(mut reader: R) -> Result<Self> {
410        let class_and_version = track!(reader.read_u8())?;
411        let version = class_and_version >> 4;
412        let class = track!(DatatypeClass::try_from(class_and_version & 0b0000_1111))?;
413        track_assert_eq!(version, 1, ErrorKind::Unsupported);
414
415        let bit_field = track!(reader.read_u24())?;
416        let size = track!(reader.read_u32())?;
417
418        match class {
419            DatatypeClass::FixedPoint => {
420                track!(FixedPointDatatype::from_reader(bit_field, size, reader))
421                    .map(DatatypeMessage::FixedPoint)
422            }
423            DatatypeClass::FloatingPoint => {
424                track!(FloatingPointDatatype::from_reader(bit_field, size, reader))
425                    .map(DatatypeMessage::FloatingPoint)
426            }
427            _ => track_panic!(ErrorKind::Unsupported; class),
428        }
429    }
430}
431
432/// type=0x05
433#[derive(Debug, Clone)]
434pub struct FillValueMessage {
435    space_allocation_time: u8,
436    fill_value_write_time: u8,
437    fill_value: Option<Vec<u8>>,
438}
439impl FillValueMessage {
440    pub fn from_reader<R: Read>(mut reader: R) -> Result<Self> {
441        let version = track!(reader.read_u8())?;
442        track_assert_eq!(version, 2, ErrorKind::Unsupported);
443
444        let space_allocation_time = track!(reader.read_u8())?;
445        let fill_value_write_time = track!(reader.read_u8())?;
446        let fill_value_defined = track!(reader.read_u8())?;
447        let fill_value = if fill_value_defined == 1 {
448            let size = track!(reader.read_u32())?;
449            let fill_value = track!(reader.read_vec(size as usize))?;
450            Some(fill_value)
451        } else {
452            None
453        };
454        Ok(Self {
455            space_allocation_time,
456            fill_value_write_time,
457            fill_value,
458        })
459    }
460}
461
462#[derive(Debug, Clone)]
463pub enum Layout {
464    Contiguous { address: u64, size: u64 },
465}
466impl Layout {
467    pub fn from_reader<R: Read>(class: u8, mut reader: R) -> Result<Self> {
468        match class {
469            0 => track_panic!(ErrorKind::Unsupported),
470            1 => {
471                let address = track!(reader.read_u64())?;
472                let size = track!(reader.read_u64())?;
473                Ok(Layout::Contiguous { address, size })
474            }
475            2 => track_panic!(ErrorKind::Unsupported),
476            _ => track_panic!(ErrorKind::InvalidFile, "Unknown layout class: {}", class),
477        }
478    }
479}
480
481/// type=0x08
482#[derive(Debug, Clone)]
483pub struct DataLayoutMessage {
484    layout: Layout,
485}
486impl DataLayoutMessage {
487    pub fn from_reader<R: Read>(mut reader: R) -> Result<Self> {
488        let version = track!(reader.read_u8())?;
489        track_assert_eq!(version, 3, ErrorKind::Unsupported);
490
491        let layout_class = track!(reader.read_u8())?;
492        let layout = track!(Layout::from_reader(layout_class, &mut reader))?;
493        let _padding = track!(reader.read_all())?;
494        Ok(Self { layout })
495    }
496}
497
498/// type=0x11
499#[derive(Debug, Clone)]
500pub struct SymbolTableMessage {
501    pub b_tree_address: u64,
502    pub local_heap_address: u64,
503}
504impl SymbolTableMessage {
505    pub fn from_reader<R: Read>(mut reader: R) -> Result<Self> {
506        Ok(Self {
507            b_tree_address: track!(reader.read_u64())?,
508            local_heap_address: track!(reader.read_u64())?,
509        })
510    }
511}
512
513/// type=0x12
514#[derive(Debug, Clone)]
515pub struct ObjectModificationTimeMessage {
516    unixtime_seconds: u32,
517}
518impl ObjectModificationTimeMessage {
519    pub fn from_reader<R: Read>(mut reader: R) -> Result<Self> {
520        let version = track!(reader.read_u8())?;
521        track_assert_eq!(version, 1, ErrorKind::Unsupported);
522        track!(reader.skip(3))?;
523
524        let unixtime_seconds = track!(reader.read_u32())?;
525        Ok(Self { unixtime_seconds })
526    }
527}
528
529#[derive(Debug, Clone)]
530pub enum Message {
531    Nil(NilMessage),
532    Dataspace(DataspaceMessage),
533    // LinkInfo,
534    Datatype(DatatypeMessage),
535    // FillValueOld,
536    FillValue(FillValueMessage),
537    // Link,
538    // ExternalDataFile,
539    DataLayout(DataLayoutMessage),
540    // Bogus,
541    // GroupInfo,
542    // FilePipeline,
543    // Attribute,
544    // ObjectComment,
545    // ObjectModificationTimeOld,
546    // SharedMessageTable,
547    // ObjectHeaderContinuation,
548    SymbolTable(SymbolTableMessage),
549    ObjectModificationTime(ObjectModificationTimeMessage),
550    // BTreeKValues,
551    // DriverInfo,
552    // AttributeInfo,
553    // ObjectReferenceCount,
554}
555
556#[cfg(test)]
557mod tests {
558    use super::*;
559    use trackable::result::TopLevelResult;
560
561    #[test]
562    fn floating_point_decode_works() -> TopLevelResult {
563        let datatype = FloatingPointDatatype {
564            size: 4,
565            endian: Endian::Little,
566            low_padding_bit: 0,
567            high_padding_bit: 0,
568            internal_padding_bit: 0,
569            mantissa_norm: MantissaNorm::ImpliedToBeSet,
570            sign_location: 31,
571            bit_offset: 0,
572            bit_precision: 32,
573            exponent_location: 23,
574            exponent_size: 8,
575            mantissa_location: 0,
576            mantissa_size: 23,
577            exponent_bias: 127,
578        };
579        let bytes = [166, 73, 90, 67];
580
581        let item = track!(datatype.decode(&bytes[..]))?;
582        assert_eq!(item, 218.28768920898438);
583        Ok(())
584    }
585}
hdf5file/lowlevel/level2.rs

hdf5file/lowlevel/
level2.rs
