asdf_core/block/
header.rs

1//! Binary block headers.
2//!
3//! Unlike the YAML tree, this layer is byte-exact by specification. Every
4//! field is big-endian, and `header_size` is authoritative: the standard
5//! requires readers to obey it rather than assume 48, because a writer may
6//! enlarge the header to align block data to a filesystem boundary.
7
8use crate::error::{Error, ErrorCode, Result, err};
9
10/// The bytes that introduce every block: `0xd3` followed by `BLK`.
11pub const BLOCK_MAGIC: &[u8; 4] = b"\xd3BLK";
12
13/// Length of [`BLOCK_MAGIC`].
14pub const BLOCK_MAGIC_SIZE: usize = 4;
15
16/// The smallest header the standard permits, measured the way `header_size`
17/// measures it: excluding the magic and the `header_size` field itself.
18pub const BLOCK_HEADER_SIZE: usize = 48;
19
20/// Size of the full fixed prologue: magic, `header_size`, and a minimal header.
21pub const BLOCK_HEADER_FULL_SIZE: usize = BLOCK_HEADER_SIZE + BLOCK_MAGIC_SIZE + 2;
22
23/// Size of the compression name field.
24pub const COMPRESSION_FIELD_SIZE: usize = 4;
25
26/// Size of the MD5 digest stored in the header.
27pub const CHECKSUM_SIZE: usize = 16;
28
29/// The maximum a block header may occupy, per the standard's stated limits.
30pub const MAX_BLOCK_HEADER_SIZE: usize = 65536;
31
32// Field offsets, measured from just after `header_size`.
33const OFF_FLAGS: usize = 0;
34const OFF_COMPRESSION: usize = 4;
35const OFF_ALLOCATED_SIZE: usize = 8;
36const OFF_USED_SIZE: usize = 16;
37const OFF_DATA_SIZE: usize = 24;
38const OFF_CHECKSUM: usize = 32;
39
40/// Set when the block extends to the end of the file.
41///
42/// A streamed block ignores the three size fields, must be the last block in
43/// the file, and forbids a block index.
44pub const FLAG_STREAMED: u32 = 0x1;
45
46/// A decoded block header.
47#[derive(Clone, PartialEq, Eq, Debug)]
48pub struct BlockHeader {
49    /// The header size as recorded in the file, excluding the magic and this
50    /// field. Preserved so a re-emitted block keeps any padding it had.
51    pub header_size: u16,
52    /// Flag bits; see [`FLAG_STREAMED`].
53    pub flags: u32,
54    /// The compression name, `\0`-padded to four bytes in the file.
55    pub compression: [u8; COMPRESSION_FIELD_SIZE],
56    /// Space reserved for the block's data, excluding the header.
57    pub allocated_size: u64,
58    /// Bytes actually used on disk, excluding the header.
59    pub used_size: u64,
60    /// Size of the data once decoded. Equal to `used_size` when uncompressed.
61    pub data_size: u64,
62    /// MD5 of the used data. All-zero means "do not verify".
63    pub checksum: [u8; CHECKSUM_SIZE],
64}
65
66impl Default for BlockHeader {
67    fn default() -> Self {
68        Self {
69            header_size: BLOCK_HEADER_SIZE as u16,
70            flags: 0,
71            compression: [0; COMPRESSION_FIELD_SIZE],
72            allocated_size: 0,
73            used_size: 0,
74            data_size: 0,
75            checksum: [0; CHECKSUM_SIZE],
76        }
77    }
78}
79
80/// Does this buffer start with the block magic?
81pub fn is_block_magic(buf: &[u8]) -> bool {
82    buf.len() >= BLOCK_MAGIC_SIZE && &buf[..BLOCK_MAGIC_SIZE] == BLOCK_MAGIC
83}
84
85fn be_u16(b: &[u8]) -> u16 {
86    u16::from_be_bytes([b[0], b[1]])
87}
88fn be_u32(b: &[u8]) -> u32 {
89    u32::from_be_bytes([b[0], b[1], b[2], b[3]])
90}
91fn be_u64(b: &[u8]) -> u64 {
92    u64::from_be_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]])
93}
94
95impl BlockHeader {
96    /// Decode a header from a buffer positioned at the block magic.
97    ///
98    /// Returns the header and the number of bytes it occupied, so the caller
99    /// can find the data that follows.
100    pub fn parse(buf: &[u8]) -> Result<(Self, usize)> {
101        if buf.len() < BLOCK_MAGIC_SIZE + 2 {
102            return Err(err!(UnexpectedEof, "truncated block header"));
103        }
104        if !is_block_magic(buf) {
105            return Err(Error::new(
106                ErrorCode::BlockMagicMismatch,
107                "block magic bytes did not match",
108            ));
109        }
110
111        let header_size = be_u16(&buf[BLOCK_MAGIC_SIZE..]);
112        let hs = usize::from(header_size);
113
114        if hs < BLOCK_HEADER_SIZE {
115            return Err(err!(
116                InvalidBlockHeader,
117                "block header_size {hs} is below the {BLOCK_HEADER_SIZE}-byte minimum"
118            ));
119        }
120        if hs > MAX_BLOCK_HEADER_SIZE {
121            return Err(err!(
122                InvalidBlockHeader,
123                "block header_size {hs} exceeds the {MAX_BLOCK_HEADER_SIZE}-byte limit"
124            ));
125        }
126
127        let total = BLOCK_MAGIC_SIZE + 2 + hs;
128        if buf.len() < total {
129            return Err(err!(UnexpectedEof, "truncated block header: need {total} bytes"));
130        }
131
132        // Fields are read at their fixed offsets within the declared header;
133        // anything beyond `OFF_CHECKSUM + CHECKSUM_SIZE` is padding.
134        let f = &buf[BLOCK_MAGIC_SIZE + 2..total];
135
136        let mut compression = [0u8; COMPRESSION_FIELD_SIZE];
137        compression.copy_from_slice(&f[OFF_COMPRESSION..OFF_COMPRESSION + COMPRESSION_FIELD_SIZE]);
138
139        let mut checksum = [0u8; CHECKSUM_SIZE];
140        checksum.copy_from_slice(&f[OFF_CHECKSUM..OFF_CHECKSUM + CHECKSUM_SIZE]);
141
142        let header = BlockHeader {
143            header_size,
144            flags: be_u32(&f[OFF_FLAGS..]),
145            compression,
146            allocated_size: be_u64(&f[OFF_ALLOCATED_SIZE..]),
147            used_size: be_u64(&f[OFF_USED_SIZE..]),
148            data_size: be_u64(&f[OFF_DATA_SIZE..]),
149            checksum,
150        };
151
152        header.validate()?;
153        Ok((header, total))
154    }
155
156    /// Check the internal consistency the standard requires.
157    fn validate(&self) -> Result<()> {
158        if self.is_streamed() {
159            // The size fields are explicitly ignored for a streamed block.
160            return Ok(());
161        }
162        if self.compression_name().is_empty() && self.data_size != self.used_size {
163            return Err(err!(
164                InvalidBlockHeader,
165                "uncompressed block has data_size {} but used_size {}",
166                self.data_size,
167                self.used_size
168            ));
169        }
170        if self.allocated_size < self.used_size {
171            return Err(err!(
172                InvalidBlockHeader,
173                "block allocated_size {} is smaller than used_size {}",
174                self.allocated_size,
175                self.used_size
176            ));
177        }
178        Ok(())
179    }
180
181    /// Encode the header, including magic, into `out`.
182    ///
183    /// Any `header_size` beyond the fields is written as zero padding, so a
184    /// block that was read with an enlarged header re-emits at the same size.
185    pub fn write(&self, out: &mut Vec<u8>) {
186        let hs = usize::from(self.header_size).max(BLOCK_HEADER_SIZE);
187        out.extend_from_slice(BLOCK_MAGIC);
188        out.extend_from_slice(&(hs as u16).to_be_bytes());
189
190        let start = out.len();
191        out.resize(start + hs, 0);
192        let f = &mut out[start..start + hs];
193
194        f[OFF_FLAGS..OFF_FLAGS + 4].copy_from_slice(&self.flags.to_be_bytes());
195        f[OFF_COMPRESSION..OFF_COMPRESSION + COMPRESSION_FIELD_SIZE]
196            .copy_from_slice(&self.compression);
197        f[OFF_ALLOCATED_SIZE..OFF_ALLOCATED_SIZE + 8]
198            .copy_from_slice(&self.allocated_size.to_be_bytes());
199        f[OFF_USED_SIZE..OFF_USED_SIZE + 8].copy_from_slice(&self.used_size.to_be_bytes());
200        f[OFF_DATA_SIZE..OFF_DATA_SIZE + 8].copy_from_slice(&self.data_size.to_be_bytes());
201        f[OFF_CHECKSUM..OFF_CHECKSUM + CHECKSUM_SIZE].copy_from_slice(&self.checksum);
202    }
203
204    /// The number of bytes this header occupies in the file.
205    pub fn on_disk_size(&self) -> usize {
206        BLOCK_MAGIC_SIZE + 2 + usize::from(self.header_size)
207    }
208
209    /// Whether the streamed flag is set.
210    pub fn is_streamed(&self) -> bool {
211        self.flags & FLAG_STREAMED != 0
212    }
213
214    /// The compression name with its `\0` padding removed.
215    ///
216    /// An empty name means the block is uncompressed.
217    pub fn compression_name(&self) -> &str {
218        let end = self.compression.iter().position(|b| *b == 0).unwrap_or(COMPRESSION_FIELD_SIZE);
219        core::str::from_utf8(&self.compression[..end]).unwrap_or("")
220    }
221
222    /// Set the compression name, which must fit in four bytes.
223    pub fn set_compression(&mut self, name: &str) -> Result<()> {
224        let bytes = name.as_bytes();
225        if bytes.len() > COMPRESSION_FIELD_SIZE {
226            return Err(err!(
227                UnknownCompression,
228                "compression name {name:?} exceeds {COMPRESSION_FIELD_SIZE} bytes"
229            ));
230        }
231        self.compression = [0; COMPRESSION_FIELD_SIZE];
232        self.compression[..bytes.len()].copy_from_slice(bytes);
233        Ok(())
234    }
235
236    /// Whether a checksum is recorded. All-zero means "do not verify".
237    pub fn has_checksum(&self) -> bool {
238        self.checksum.iter().any(|b| *b != 0)
239    }
240}
241
242#[cfg(test)]
243mod tests {
244    use super::*;
245
246    fn sample() -> BlockHeader {
247        let mut h =
248            BlockHeader { allocated_size: 64, used_size: 64, data_size: 64, ..Default::default() };
249        h.set_compression("").unwrap();
250        h
251    }
252
253    #[test]
254    fn magic_is_the_documented_byte_sequence() {
255        assert_eq!(BLOCK_MAGIC, &[0xd3, 0x42, 0x4c, 0x4b]);
256        assert_eq!(BLOCK_MAGIC, b"\xd3BLK");
257    }
258
259    #[test]
260    fn round_trips_through_bytes() {
261        let h = sample();
262        let mut buf = Vec::new();
263        h.write(&mut buf);
264        assert_eq!(buf.len(), BLOCK_HEADER_FULL_SIZE);
265
266        let (parsed, consumed) = BlockHeader::parse(&buf).unwrap();
267        assert_eq!(parsed, h);
268        assert_eq!(consumed, BLOCK_HEADER_FULL_SIZE);
269    }
270
271    #[test]
272    fn fields_are_big_endian_at_documented_offsets() {
273        let mut h = sample();
274        h.flags = 0x0000_0001;
275        h.allocated_size = 0x0102_0304_0506_0708;
276        let mut buf = Vec::new();
277        h.write(&mut buf);
278
279        assert_eq!(&buf[0..4], BLOCK_MAGIC);
280        assert_eq!(&buf[4..6], &[0x00, 0x30]); // header_size == 48
281        assert_eq!(&buf[6..10], &[0, 0, 0, 1]); // flags, big-endian
282        assert_eq!(&buf[14..22], &[0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]);
283    }
284
285    #[test]
286    fn honours_an_enlarged_header_size() {
287        // A writer may pad the header to align block data; readers must obey
288        // header_size rather than assume 48.
289        let mut h = sample();
290        h.header_size = 96;
291        let mut buf = Vec::new();
292        h.write(&mut buf);
293        assert_eq!(buf.len(), BLOCK_MAGIC_SIZE + 2 + 96);
294
295        let (parsed, consumed) = BlockHeader::parse(&buf).unwrap();
296        assert_eq!(parsed.header_size, 96);
297        assert_eq!(consumed, BLOCK_MAGIC_SIZE + 2 + 96);
298        assert_eq!(parsed.allocated_size, 64, "fields still read at fixed offsets");
299    }
300
301    #[test]
302    fn rejects_bad_magic() {
303        let mut buf = Vec::new();
304        sample().write(&mut buf);
305        buf[1] = b'X';
306        let e = BlockHeader::parse(&buf).unwrap_err();
307        assert_eq!(e.code(), ErrorCode::BlockMagicMismatch);
308    }
309
310    #[test]
311    fn rejects_undersized_header_size() {
312        let mut buf = Vec::new();
313        sample().write(&mut buf);
314        buf[4..6].copy_from_slice(&40u16.to_be_bytes());
315        let e = BlockHeader::parse(&buf).unwrap_err();
316        assert_eq!(e.code(), ErrorCode::InvalidBlockHeader);
317    }
318
319    #[test]
320    fn rejects_truncated_input() {
321        let mut buf = Vec::new();
322        sample().write(&mut buf);
323        buf.truncate(20);
324        let e = BlockHeader::parse(&buf).unwrap_err();
325        assert_eq!(e.code(), ErrorCode::UnexpectedEof);
326    }
327
328    #[test]
329    fn uncompressed_block_must_have_matching_sizes() {
330        let mut h = sample();
331        h.data_size = 99;
332        let mut buf = Vec::new();
333        h.write(&mut buf);
334        let e = BlockHeader::parse(&buf).unwrap_err();
335        assert_eq!(e.code(), ErrorCode::InvalidBlockHeader);
336    }
337
338    #[test]
339    fn compressed_block_may_have_differing_sizes() {
340        let mut h =
341            BlockHeader { allocated_size: 20, used_size: 20, data_size: 64, ..Default::default() };
342        h.set_compression("zlib").unwrap();
343        let mut buf = Vec::new();
344        h.write(&mut buf);
345        let (parsed, _) = BlockHeader::parse(&buf).unwrap();
346        assert_eq!(parsed.compression_name(), "zlib");
347        assert_eq!(parsed.data_size, 64);
348    }
349
350    #[test]
351    fn streamed_blocks_skip_size_validation() {
352        // The standard says the size fields are ignored when STREAMED is set.
353        let mut h = BlockHeader { flags: FLAG_STREAMED, ..Default::default() };
354        h.data_size = 12345;
355        h.used_size = 0;
356        h.allocated_size = 0;
357        let mut buf = Vec::new();
358        h.write(&mut buf);
359        let (parsed, _) = BlockHeader::parse(&buf).unwrap();
360        assert!(parsed.is_streamed());
361    }
362
363    #[test]
364    fn compression_names_pad_and_trim() {
365        let mut h = sample();
366        h.set_compression("lz4").unwrap();
367        assert_eq!(h.compression, [b'l', b'z', b'4', 0]);
368        assert_eq!(h.compression_name(), "lz4");
369
370        h.set_compression("bzp2").unwrap();
371        assert_eq!(h.compression_name(), "bzp2");
372
373        assert!(h.set_compression("toolong").is_err());
374    }
375
376    #[test]
377    fn checksum_presence() {
378        let mut h = sample();
379        assert!(!h.has_checksum());
380        h.checksum[0] = 1;
381        assert!(h.has_checksum());
382    }
383
384    #[test]
385    fn allocated_size_must_cover_used_size() {
386        let mut h =
387            BlockHeader { allocated_size: 8, used_size: 64, data_size: 64, ..Default::default() };
388        h.set_compression("").unwrap();
389        let mut buf = Vec::new();
390        h.write(&mut buf);
391        assert_eq!(BlockHeader::parse(&buf).unwrap_err().code(), ErrorCode::InvalidBlockHeader);
392    }
393}
asdf_core/block/header.rs

asdf_core/block/
header.rs
