rust_hdf5/format/messages/
external_file_list.rs

1//! External Data Files message (type 0x0007) — H5O_EFL_ID.
2//!
3//! A dataset with this message stores its raw data outside the HDF5 file:
4//! the Data Layout message (type 0x0008) still declares `Contiguous`
5//! storage, but with its address left undefined (H5Dlayout.c switches the
6//! dataset's layout ops to `H5D_LOPS_EFL` — H5Defl.c — whenever this message
7//! is present, overriding the layout message's own storage). The dataset's
8//! logical byte range is the concatenation of every slot's reserved region,
9//! in slot order.
10//!
11//! Binary layout (version 1, `H5O__efl_decode` in H5Oefl.c):
12//! ```text
13//! version: 1 byte (= 1)
14//! reserved: 3 bytes
15//! nalloc: u16 LE (allocated slot count, > 0)
16//! nused:  u16 LE (in-use slot count, <= nalloc)
17//! heap_addr: sizeof_addr bytes (local heap holding the slot names)
18//! nused * {
19//!     name_offset: sizeof_size bytes (offset into the local heap)
20//!     offset:      sizeof_size bytes (byte offset within the named file)
21//!     size:        sizeof_size bytes (bytes reserved for this slot; the
22//!                  all-ones sentinel marks an unlimited/growable slot)
23//! }
24//! ```
25//!
26//! A slot's name is stored as an offset into the local heap at `heap_addr`,
27//! not inline — resolving it needs a second on-disk read (the heap header,
28//! then its data block), which decode does not perform; the reader does
29//! that once at dataset-discovery time; see
30//! [`crate::format::local_heap`].
31
32use crate::format::bytes::{read_le_addr as read_addr, read_le_uint as read_uint};
33use crate::format::{FormatContext, FormatError, FormatResult, UNDEF_ADDR};
34
35const VERSION: u8 = 1;
36
37/// The declared-size sentinel marking a slot as unlimited/growable
38/// (`H5O_EFL_UNLIMITED` in H5Oprivate.h, numerically `HSIZE_UNDEF` — the same
39/// all-ones pattern as [`UNDEF_ADDR`]). Only the last slot may carry it, and
40/// only for a dataset with an unlimited dataspace (`H5D__efl_construct`).
41pub const UNLIMITED: u64 = u64::MAX;
42
43/// One external-file slot, before its name is resolved through the local
44/// heap.
45#[derive(Debug, Clone, Copy, PartialEq, Eq)]
46pub struct ExternalFileSlot {
47    /// Offset of the slot's name within the local heap at
48    /// [`ExternalFileListMessage::heap_addr`].
49    pub name_offset: u64,
50    /// Byte offset within the named file where this slot's reserved
51    /// region begins.
52    pub offset: u64,
53    /// Bytes reserved for this slot. The all-ones sentinel (`u64::MAX`,
54    /// `H5O_EFL_UNLIMITED` in H5Oprivate.h) marks the last slot as
55    /// unlimited/growable.
56    pub size: u64,
57}
58
59/// Decoded External Data Files message.
60#[derive(Debug, Clone, PartialEq, Eq)]
61pub struct ExternalFileListMessage {
62    /// Address of the local heap holding every slot's name.
63    pub heap_addr: u64,
64    /// In-use slots, in the order the dataset's logical byte range
65    /// concatenates them.
66    pub slots: Vec<ExternalFileSlot>,
67}
68
69impl ExternalFileListMessage {
70    /// Encode the message (`H5O__efl_encode`).
71    ///
72    /// The allocated-slot count is written as the in-use one — upstream
73    /// encodes `nused` into both fields ("yes, twice"), so a message it wrote
74    /// never declares spare slots however many the property list reserved.
75    pub fn encode(&self, ctx: &FormatContext) -> Vec<u8> {
76        let sa = ctx.sizeof_addr as usize;
77        let ss = ctx.sizeof_size as usize;
78        let nused = self.slots.len() as u16;
79        let mut buf = Vec::with_capacity(8 + sa + self.slots.len() * 3 * ss);
80        buf.push(VERSION);
81        buf.extend_from_slice(&[0u8; 3]); // reserved
82        buf.extend_from_slice(&nused.to_le_bytes()); // nalloc
83        buf.extend_from_slice(&nused.to_le_bytes()); // nused
84        buf.extend_from_slice(&self.heap_addr.to_le_bytes()[..sa]);
85        for slot in &self.slots {
86            buf.extend_from_slice(&slot.name_offset.to_le_bytes()[..ss]);
87            buf.extend_from_slice(&slot.offset.to_le_bytes()[..ss]);
88            buf.extend_from_slice(&slot.size.to_le_bytes()[..ss]);
89        }
90        buf
91    }
92
93    pub fn decode(buf: &[u8], ctx: &FormatContext) -> FormatResult<(Self, usize)> {
94        let sa = ctx.sizeof_addr as usize;
95        let ss = ctx.sizeof_size as usize;
96
97        if buf.len() < 4 + 4 {
98            return Err(FormatError::BufferTooShort {
99                needed: 4 + 4,
100                available: buf.len(),
101            });
102        }
103        let version = buf[0];
104        if version != VERSION {
105            return Err(FormatError::InvalidVersion(version));
106        }
107        // buf[1..4] reserved
108        let nalloc = u16::from_le_bytes([buf[4], buf[5]]);
109        let nused = u16::from_le_bytes([buf[6], buf[7]]);
110        if nalloc == 0 {
111            return Err(FormatError::InvalidData(
112                "external file list message declares zero allocated slots".into(),
113            ));
114        }
115        if nused > nalloc {
116            return Err(FormatError::InvalidData(format!(
117                "external file list message has {nused} in-use slots but only {nalloc} allocated"
118            )));
119        }
120
121        let mut pos = 8;
122        if buf.len() < pos + sa {
123            return Err(FormatError::BufferTooShort {
124                needed: pos + sa,
125                available: buf.len(),
126            });
127        }
128        let heap_addr = read_addr(&buf[pos..], sa);
129        pos += sa;
130        if heap_addr == UNDEF_ADDR {
131            return Err(FormatError::InvalidData(
132                "external file list message has an undefined local heap address".into(),
133            ));
134        }
135
136        let slot_len = ss * 3;
137        let mut slots = Vec::with_capacity(nused as usize);
138        for _ in 0..nused {
139            if buf.len() < pos + slot_len {
140                return Err(FormatError::BufferTooShort {
141                    needed: pos + slot_len,
142                    available: buf.len(),
143                });
144            }
145            let name_offset = read_uint(&buf[pos..], ss);
146            pos += ss;
147            let offset = read_uint(&buf[pos..], ss);
148            pos += ss;
149            let size = read_uint(&buf[pos..], ss);
150            pos += ss;
151            slots.push(ExternalFileSlot {
152                name_offset,
153                offset,
154                size,
155            });
156        }
157
158        Ok((Self { heap_addr, slots }, pos))
159    }
160}
161
162// ======================================================================= tests
163
164#[cfg(test)]
165mod tests {
166    use super::*;
167
168    fn ctx8() -> FormatContext {
169        FormatContext {
170            sizeof_addr: 8,
171            sizeof_size: 8,
172        }
173    }
174
175    /// Byte-for-byte the message `h5debug` reported for an h5py-written
176    /// single-slot external dataset: heap at 1072, one slot named at heap
177    /// offset 8, file offset 0, 64 bytes reserved.
178    fn single_slot_buf() -> Vec<u8> {
179        let mut buf = vec![VERSION, 0, 0, 0];
180        buf.extend_from_slice(&1u16.to_le_bytes()); // nalloc
181        buf.extend_from_slice(&1u16.to_le_bytes()); // nused
182        buf.extend_from_slice(&1072u64.to_le_bytes()); // heap_addr
183        buf.extend_from_slice(&8u64.to_le_bytes()); // name_offset
184        buf.extend_from_slice(&0u64.to_le_bytes()); // offset
185        buf.extend_from_slice(&64u64.to_le_bytes()); // size
186        buf
187    }
188
189    #[test]
190    fn decode_single_slot() {
191        let buf = single_slot_buf();
192        let (msg, consumed) = ExternalFileListMessage::decode(&buf, &ctx8()).unwrap();
193        assert_eq!(consumed, buf.len());
194        assert_eq!(msg.heap_addr, 1072);
195        assert_eq!(
196            msg.slots,
197            vec![ExternalFileSlot {
198                name_offset: 8,
199                offset: 0,
200                size: 64,
201            }]
202        );
203    }
204
205    #[test]
206    fn decode_multi_slot() {
207        let mut buf = vec![VERSION, 0, 0, 0];
208        buf.extend_from_slice(&2u16.to_le_bytes()); // nalloc
209        buf.extend_from_slice(&2u16.to_le_bytes()); // nused
210        buf.extend_from_slice(&2000u64.to_le_bytes()); // heap_addr
211        buf.extend_from_slice(&8u64.to_le_bytes());
212        buf.extend_from_slice(&0u64.to_le_bytes());
213        buf.extend_from_slice(&32u64.to_le_bytes());
214        buf.extend_from_slice(&20u64.to_le_bytes());
215        buf.extend_from_slice(&0u64.to_le_bytes());
216        buf.extend_from_slice(&32u64.to_le_bytes());
217        let (msg, consumed) = ExternalFileListMessage::decode(&buf, &ctx8()).unwrap();
218        assert_eq!(consumed, buf.len());
219        assert_eq!(msg.slots.len(), 2);
220        assert_eq!(msg.slots[0].size, 32);
221        assert_eq!(msg.slots[1].name_offset, 20);
222    }
223
224    #[test]
225    fn decode_rejects_bad_version() {
226        let mut buf = single_slot_buf();
227        buf[0] = 2;
228        let err = ExternalFileListMessage::decode(&buf, &ctx8()).unwrap_err();
229        assert!(matches!(err, FormatError::InvalidVersion(2)));
230    }
231
232    #[test]
233    fn decode_rejects_zero_nalloc() {
234        let mut buf = single_slot_buf();
235        buf[4] = 0;
236        buf[5] = 0;
237        let err = ExternalFileListMessage::decode(&buf, &ctx8()).unwrap_err();
238        assert!(matches!(err, FormatError::InvalidData(_)));
239    }
240
241    #[test]
242    fn decode_rejects_nused_over_nalloc() {
243        let mut buf = single_slot_buf();
244        buf[6] = 2; // nused = 2 > nalloc = 1
245        let err = ExternalFileListMessage::decode(&buf, &ctx8()).unwrap_err();
246        assert!(matches!(err, FormatError::InvalidData(_)));
247    }
248
249    #[test]
250    fn decode_rejects_undefined_heap_addr() {
251        let mut buf = single_slot_buf();
252        buf[8..16].copy_from_slice(&[0xFFu8; 8]);
253        let err = ExternalFileListMessage::decode(&buf, &ctx8()).unwrap_err();
254        assert!(matches!(err, FormatError::InvalidData(_)));
255    }
256
257    #[test]
258    fn decode_truncated() {
259        let buf = single_slot_buf();
260        let err = ExternalFileListMessage::decode(&buf[..20], &ctx8()).unwrap_err();
261        assert!(matches!(err, FormatError::BufferTooShort { .. }));
262    }
263
264    #[test]
265    fn decode_ctx4() {
266        let ctx = FormatContext {
267            sizeof_addr: 4,
268            sizeof_size: 4,
269        };
270        let mut buf = vec![VERSION, 0, 0, 0];
271        buf.extend_from_slice(&1u16.to_le_bytes());
272        buf.extend_from_slice(&1u16.to_le_bytes());
273        buf.extend_from_slice(&0x800u32.to_le_bytes()); // heap_addr
274        buf.extend_from_slice(&8u32.to_le_bytes());
275        buf.extend_from_slice(&0u32.to_le_bytes());
276        buf.extend_from_slice(&64u32.to_le_bytes());
277        let (msg, consumed) = ExternalFileListMessage::decode(&buf, &ctx).unwrap();
278        assert_eq!(consumed, buf.len());
279        assert_eq!(msg.heap_addr, 0x800);
280        assert_eq!(msg.slots[0].size, 64);
281    }
282
283    /// The all-ones size sentinel (`H5O_EFL_UNLIMITED`) decodes as a plain
284    /// `u64::MAX` slot size rather than being special-cased at this layer —
285    /// callers that cannot support a growable slot detect and reject it
286    /// explicitly instead of this message type silently reinterpreting it.
287    #[test]
288    fn decode_preserves_unlimited_sentinel() {
289        let mut buf = single_slot_buf();
290        let last = buf.len() - 8;
291        buf[last..].copy_from_slice(&[0xFFu8; 8]);
292        let (msg, _) = ExternalFileListMessage::decode(&buf, &ctx8()).unwrap();
293        assert_eq!(msg.slots[0].size, UNLIMITED);
294    }
295
296    /// The encoder reproduces the h5py-written message this module's fixture
297    /// was captured from, byte for byte.
298    #[test]
299    fn encode_matches_the_captured_single_slot_message() {
300        let msg = ExternalFileListMessage {
301            heap_addr: 1072,
302            slots: vec![ExternalFileSlot {
303                name_offset: 8,
304                offset: 0,
305                size: 64,
306            }],
307        };
308        assert_eq!(msg.encode(&ctx8()), single_slot_buf());
309    }
310
311    #[test]
312    fn encode_roundtrips_multi_slot_at_ctx4() {
313        let ctx = FormatContext {
314            sizeof_addr: 4,
315            sizeof_size: 4,
316        };
317        let msg = ExternalFileListMessage {
318            heap_addr: 0x800,
319            slots: vec![
320                ExternalFileSlot {
321                    name_offset: 8,
322                    offset: 0,
323                    size: 32,
324                },
325                ExternalFileSlot {
326                    name_offset: 20,
327                    offset: 16,
328                    size: 32,
329                },
330            ],
331        };
332        let buf = msg.encode(&ctx);
333        let (back, consumed) = ExternalFileListMessage::decode(&buf, &ctx).unwrap();
334        assert_eq!(consumed, buf.len());
335        assert_eq!(back, msg);
336    }
337}
rust_hdf5/format/messages/external_file_list.rs

rust_hdf5/format/messages/
external_file_list.rs
