netcdf_reader/nc4/
types.rs

1//! Map HDF5 datatypes to NetCDF-4 types.
2//!
3//! HDF5 has a richer type system than NetCDF-4. This module maps the subset
4//! of HDF5 types that are valid in NetCDF-4 to `NcType`:
5//!
6//! | HDF5 Datatype                 | NcType   |
7//! |-------------------------------|----------|
8//! | FixedPoint { size=1, signed } | Byte     |
9//! | FixedPoint { size=1, !signed} | UByte    |
10//! | FixedPoint { size=2, signed } | Short    |
11//! | FixedPoint { size=2, !signed} | UShort   |
12//! | FixedPoint { size=4, signed } | Int      |
13//! | FixedPoint { size=4, !signed} | UInt     |
14//! | FixedPoint { size=8, signed } | Int64    |
15//! | FixedPoint { size=8, !signed} | UInt64   |
16//! | FloatingPoint { size=4 }      | Float    |
17//! | FloatingPoint { size=8 }      | Double   |
18//! | String (any)                  | String   |
19//! | Enum { base=byte }            | Enum     |
20//! | Compound { .. }               | Compound |
21//! | Opaque { .. }                 | Opaque   |
22//! | Array { base, dims }          | Array    |
23//! | VarLen { kind=String, base=u8 } | String*  |
24//! | VarLen { base }               | VLen     |
25//!
26//! * Some NetCDF-4 string variables are stored as HDF5 vlen bytes.
27
28use hdf5_reader::messages::datatype::{Datatype, StringSize, VarLenKind};
29use hdf5_reader::ByteOrder;
30
31use crate::error::{Error, Result};
32use crate::types::{NcCompoundField, NcEnumMember, NcIntegerValue, NcType};
33
34/// Map an HDF5 datatype to a NetCDF type.
35pub fn hdf5_to_nc_type(dtype: &Datatype) -> Result<NcType> {
36    match dtype {
37        Datatype::FixedPoint { size, signed, .. } => match (size, signed) {
38            (1, true) => Ok(NcType::Byte),
39            (1, false) => Ok(NcType::UByte),
40            (2, true) => Ok(NcType::Short),
41            (2, false) => Ok(NcType::UShort),
42            (4, true) => Ok(NcType::Int),
43            (4, false) => Ok(NcType::UInt),
44            (8, true) => Ok(NcType::Int64),
45            (8, false) => Ok(NcType::UInt64),
46            _ => Err(Error::InvalidData(format!(
47                "unsupported HDF5 integer size {} for NetCDF-4",
48                size
49            ))),
50        },
51        Datatype::FloatingPoint { size, .. } => match size {
52            4 => Ok(NcType::Float),
53            8 => Ok(NcType::Double),
54            _ => Err(Error::InvalidData(format!(
55                "unsupported HDF5 float size {} for NetCDF-4",
56                size
57            ))),
58        },
59        Datatype::String {
60            size: StringSize::Fixed(1),
61            ..
62        } => Ok(NcType::Char),
63        Datatype::String { .. } => Ok(NcType::String),
64        Datatype::Enum { base, members } => Ok(NcType::Enum {
65            base: Box::new(hdf5_to_nc_type(base)?),
66            members: members
67                .iter()
68                .map(|member| {
69                    Ok(NcEnumMember {
70                        name: member.name.clone(),
71                        value: decode_enum_integer(base, &member.value)?,
72                    })
73                })
74                .collect::<Result<Vec<_>>>()?,
75        }),
76        Datatype::Compound { size, fields } => {
77            let mut nc_fields = Vec::with_capacity(fields.len());
78            for f in fields {
79                nc_fields.push(NcCompoundField {
80                    name: f.name.clone(),
81                    offset: f.byte_offset as u64,
82                    dtype: hdf5_to_nc_type(&f.datatype)?,
83                });
84            }
85            Ok(NcType::Compound {
86                size: *size,
87                fields: nc_fields,
88            })
89        }
90        Datatype::Opaque { size, tag } => Ok(NcType::Opaque {
91            size: *size,
92            tag: tag.clone(),
93        }),
94        Datatype::Array { base, dims } => {
95            let base_nc = hdf5_to_nc_type(base)?;
96            Ok(NcType::Array {
97                base: Box::new(base_nc),
98                dims: dims.clone(),
99            })
100        }
101        Datatype::VarLen {
102            base,
103            kind: VarLenKind::String,
104            ..
105        } if matches!(base.as_ref(), Datatype::FixedPoint { size: 1, .. }) => Ok(NcType::String),
106        Datatype::VarLen { base, .. } => {
107            let base_nc = hdf5_to_nc_type(base)?;
108            Ok(NcType::VLen {
109                base: Box::new(base_nc),
110            })
111        }
112        _ => Err(Error::InvalidData(format!(
113            "HDF5 datatype {:?} has no NetCDF-4 equivalent",
114            dtype
115        ))),
116    }
117}
118
119pub(crate) fn decode_enum_integer(base: &Datatype, bytes: &[u8]) -> Result<NcIntegerValue> {
120    match base {
121        Datatype::FixedPoint {
122            size,
123            signed,
124            byte_order,
125        } => decode_fixed_point_integer(bytes, *size, *signed, *byte_order),
126        other => Err(Error::InvalidData(format!(
127            "NetCDF-4 enum base type must be integer, got {other:?}"
128        ))),
129    }
130}
131
132pub(crate) fn decode_fixed_point_integer(
133    bytes: &[u8],
134    size: u8,
135    signed: bool,
136    byte_order: ByteOrder,
137) -> Result<NcIntegerValue> {
138    fn read<const N: usize>(bytes: &[u8], byte_order: ByteOrder) -> Result<[u8; N]> {
139        if bytes.len() < N {
140            return Err(Error::InvalidData(format!(
141                "integer value too short: need {} bytes, have {}",
142                N,
143                bytes.len()
144            )));
145        }
146        let mut out = [0u8; N];
147        out.copy_from_slice(&bytes[..N]);
148        #[cfg(target_endian = "little")]
149        if byte_order == ByteOrder::BigEndian {
150            out.reverse();
151        }
152        #[cfg(target_endian = "big")]
153        if byte_order == ByteOrder::LittleEndian {
154            out.reverse();
155        }
156        Ok(out)
157    }
158
159    match (size, signed) {
160        (1, true) => Ok(NcIntegerValue::I8(i8::from_ne_bytes(read::<1>(
161            bytes, byte_order,
162        )?))),
163        (1, false) => Ok(NcIntegerValue::U8(u8::from_ne_bytes(read::<1>(
164            bytes, byte_order,
165        )?))),
166        (2, true) => Ok(NcIntegerValue::I16(i16::from_ne_bytes(read::<2>(
167            bytes, byte_order,
168        )?))),
169        (2, false) => Ok(NcIntegerValue::U16(u16::from_ne_bytes(read::<2>(
170            bytes, byte_order,
171        )?))),
172        (4, true) => Ok(NcIntegerValue::I32(i32::from_ne_bytes(read::<4>(
173            bytes, byte_order,
174        )?))),
175        (4, false) => Ok(NcIntegerValue::U32(u32::from_ne_bytes(read::<4>(
176            bytes, byte_order,
177        )?))),
178        (8, true) => Ok(NcIntegerValue::I64(i64::from_ne_bytes(read::<8>(
179            bytes, byte_order,
180        )?))),
181        (8, false) => Ok(NcIntegerValue::U64(u64::from_ne_bytes(read::<8>(
182            bytes, byte_order,
183        )?))),
184        _ => Err(Error::InvalidData(format!(
185            "unsupported NetCDF-4 enum integer size {size}"
186        ))),
187    }
188}
189
190#[cfg(test)]
191mod tests {
192    use super::*;
193    use hdf5_reader::error::ByteOrder;
194
195    #[test]
196    fn integer_types() {
197        let bo = ByteOrder::LittleEndian;
198        assert_eq!(
199            hdf5_to_nc_type(&Datatype::FixedPoint {
200                size: 1,
201                signed: true,
202                byte_order: bo
203            })
204            .unwrap(),
205            NcType::Byte
206        );
207        assert_eq!(
208            hdf5_to_nc_type(&Datatype::FixedPoint {
209                size: 1,
210                signed: false,
211                byte_order: bo
212            })
213            .unwrap(),
214            NcType::UByte
215        );
216        assert_eq!(
217            hdf5_to_nc_type(&Datatype::FixedPoint {
218                size: 4,
219                signed: true,
220                byte_order: bo
221            })
222            .unwrap(),
223            NcType::Int
224        );
225        assert_eq!(
226            hdf5_to_nc_type(&Datatype::FixedPoint {
227                size: 8,
228                signed: false,
229                byte_order: bo
230            })
231            .unwrap(),
232            NcType::UInt64
233        );
234    }
235
236    #[test]
237    fn float_types() {
238        let bo = ByteOrder::LittleEndian;
239        assert_eq!(
240            hdf5_to_nc_type(&Datatype::FloatingPoint {
241                size: 4,
242                byte_order: bo
243            })
244            .unwrap(),
245            NcType::Float
246        );
247        assert_eq!(
248            hdf5_to_nc_type(&Datatype::FloatingPoint {
249                size: 8,
250                byte_order: bo
251            })
252            .unwrap(),
253            NcType::Double
254        );
255    }
256
257    #[test]
258    fn varlen_u8_maps_to_string() {
259        let bo = ByteOrder::LittleEndian;
260        assert_eq!(
261            hdf5_to_nc_type(&Datatype::VarLen {
262                base: Box::new(Datatype::FixedPoint {
263                    size: 1,
264                    signed: false,
265                    byte_order: bo,
266                }),
267                kind: VarLenKind::String,
268                encoding: hdf5_reader::StringEncoding::Utf8,
269                padding: hdf5_reader::StringPadding::NullTerminate,
270            })
271            .unwrap(),
272            NcType::String
273        );
274    }
275
276    #[test]
277    fn fixed_one_byte_string_maps_to_char() {
278        assert_eq!(
279            hdf5_to_nc_type(&Datatype::String {
280                size: StringSize::Fixed(1),
281                encoding: hdf5_reader::StringEncoding::Ascii,
282                padding: hdf5_reader::StringPadding::NullPad,
283            })
284            .unwrap(),
285            NcType::Char
286        );
287    }
288
289    #[test]
290    fn sequence_varlen_u8_maps_to_vlen() {
291        let bo = ByteOrder::LittleEndian;
292        assert_eq!(
293            hdf5_to_nc_type(&Datatype::VarLen {
294                base: Box::new(Datatype::FixedPoint {
295                    size: 1,
296                    signed: false,
297                    byte_order: bo,
298                }),
299                kind: VarLenKind::Sequence,
300                encoding: hdf5_reader::StringEncoding::Ascii,
301                padding: hdf5_reader::StringPadding::NullTerminate,
302            })
303            .unwrap(),
304            NcType::VLen {
305                base: Box::new(NcType::UByte)
306            }
307        );
308    }
309}
netcdf_reader/nc4/types.rs

netcdf_reader/nc4/
types.rs
