rust_hdf5/format/messages/

fill_value.rs

//! Fill value message (type 0x05) — specifies the default fill value for
//! unwritten elements.
//!
//! Three on-disk versions exist for this message and a conforming reader
//! must accept all of them (`H5O__fill_new_decode`, H5Ofill.c):
//!
//! Versions 1 & 2:
//!   Byte 0: version (1 or 2)
//!   Byte 1: space allocation time (1=early, 2=late, 3=incremental)
//!   Byte 2: fill value write time (0=on_alloc, 1=never, 2=if_set)
//!   Byte 3: fill value defined (0=undefined, non-zero=defined)
//!   [if defined != 0]: u32 LE size + `size` bytes of fill data
//!
//! Version 3 (the version this crate writes):
//!   Byte 0: version = 3
//!   Byte 1: flags — bits 0-1 allocation time, bits 2-3 fill write time,
//!           0x10 = fill value undefined, 0x20 = explicit fill value present
//!   [if 0x20 set]: u32 LE size + `size` bytes of fill data
//!
//! libhdf5 2.0.0 still writes version 2 for this message, so decoding must
//! handle every version even though encoding always emits version 3.

use crate::format::{FormatError, FormatResult};

const VERSION: u8 = 3;

// Version-3 flags byte layout (H5Ofill.c).
const FLAG_MASK_ALLOC: u8 = 0x03;
const FLAG_MASK_FILL: u8 = 0x03;
const FLAG_SHIFT_FILL: u8 = 2;
const FLAG_UNDEFINED: u8 = 0x10;
const FLAG_HAVE_VALUE: u8 = 0x20;
const FLAGS_ALL: u8 =
    FLAG_MASK_ALLOC | (FLAG_MASK_FILL << FLAG_SHIFT_FILL) | FLAG_UNDEFINED | FLAG_HAVE_VALUE;

/// Fill value message payload.
#[derive(Debug, Clone, PartialEq)]
pub struct FillValueMessage {
    /// Space allocation time: 1=early, 2=late, 3=incremental.
    pub alloc_time: u8,
    /// Fill value write time: 0=on alloc, 1=never, 2=if set.
    pub fill_write_time: u8,
    /// Fill value defined: 0=undefined, 1=default (zeros), 2=user-defined.
    pub fill_defined: u8,
    /// User-defined fill value data.  Present only when `fill_defined == 2`.
    pub fill_value: Option<Vec<u8>>,
}

impl Default for FillValueMessage {
    fn default() -> Self {
        Self {
            alloc_time: 2,      // late
            fill_write_time: 0, // on alloc
            fill_defined: 1,    // default value (zeros)
            fill_value: None,
        }
    }
}

impl FillValueMessage {
    /// A user-defined fill value.
    pub fn with_value(data: Vec<u8>) -> Self {
        Self {
            alloc_time: 2,
            fill_write_time: 0,
            fill_defined: 2,
            fill_value: Some(data),
        }
    }

    /// An undefined fill value (no fill is performed).
    pub fn undefined() -> Self {
        Self {
            alloc_time: 2,
            fill_write_time: 1, // never
            fill_defined: 0,
            fill_value: None,
        }
    }

    // ------------------------------------------------------------------ encode

    /// Encode as a version-3 fill-value message (`H5O__fill_new_encode`).
    pub fn encode(&self) -> Vec<u8> {
        let mut buf = Vec::with_capacity(10);
        buf.push(VERSION);

        let flags = (self.alloc_time & FLAG_MASK_ALLOC)
            | ((self.fill_write_time & FLAG_MASK_FILL) << FLAG_SHIFT_FILL);

        if self.fill_defined == 0 {
            // Explicitly undefined: no value follows.
            buf.push(flags | FLAG_UNDEFINED);
        } else if let Some(data) = self.fill_value.as_ref().filter(|d| !d.is_empty()) {
            // Explicit value present.
            buf.push(flags | FLAG_HAVE_VALUE);
            buf.extend_from_slice(&(data.len() as u32).to_le_bytes());
            buf.extend_from_slice(data);
        } else {
            // Defined, but no explicit value (default zero fill).
            buf.push(flags);
        }

        buf
    }

    // ------------------------------------------------------------------ decode

    /// Decode a fill-value message of version 1, 2, or 3.
    pub fn decode(buf: &[u8]) -> FormatResult<(Self, usize)> {
        if buf.is_empty() {
            return Err(FormatError::BufferTooShort {
                needed: 1,
                available: 0,
            });
        }
        match buf[0] {
            1 | 2 => Self::decode_v1v2(buf),
            3 => Self::decode_v3(buf),
            other => Err(FormatError::InvalidVersion(other)),
        }
    }

    /// Decode the version-1/2 layout (separate alloc/fill-time/defined bytes).
    fn decode_v1v2(buf: &[u8]) -> FormatResult<(Self, usize)> {
        if buf.len() < 4 {
            return Err(FormatError::BufferTooShort {
                needed: 4,
                available: buf.len(),
            });
        }
        let alloc_time = buf[1];
        let fill_write_time = buf[2];
        let defined_byte = buf[3];

        let mut pos = 4;
        let mut fill_value = None;
        if defined_byte != 0 {
            if buf.len() < pos + 4 {
                return Err(FormatError::BufferTooShort {
                    needed: pos + 4,
                    available: buf.len(),
                });
            }
            let size =
                u32::from_le_bytes([buf[pos], buf[pos + 1], buf[pos + 2], buf[pos + 3]]) as usize;
            pos += 4;
            if size > 0 {
                if buf.len() < pos + size {
                    return Err(FormatError::BufferTooShort {
                        needed: pos + size,
                        available: buf.len(),
                    });
                }
                fill_value = Some(buf[pos..pos + size].to_vec());
                pos += size;
            }
        }

        // Normalize `fill_defined` onto this crate's tri-state: an explicit
        // non-empty value is "user-defined", any other defined message is
        // "default zero fill", an undefined message is "undefined".
        let fill_defined = if fill_value.is_some() {
            2
        } else if defined_byte != 0 {
            1
        } else {
            0
        };

        Ok((
            Self {
                alloc_time,
                fill_write_time,
                fill_defined,
                fill_value,
            },
            pos,
        ))
    }

    /// Decode the version-3 layout (packed flags byte).
    fn decode_v3(buf: &[u8]) -> FormatResult<(Self, usize)> {
        if buf.len() < 2 {
            return Err(FormatError::BufferTooShort {
                needed: 2,
                available: buf.len(),
            });
        }
        let flags = buf[1];
        if flags & !FLAGS_ALL != 0 {
            return Err(FormatError::InvalidData(format!(
                "unknown flags 0x{flags:02x} in version-3 fill-value message"
            )));
        }
        let alloc_time = flags & FLAG_MASK_ALLOC;
        let fill_write_time = (flags >> FLAG_SHIFT_FILL) & FLAG_MASK_FILL;

        let mut pos = 2;
        let (fill_defined, fill_value) = if flags & FLAG_UNDEFINED != 0 {
            if flags & FLAG_HAVE_VALUE != 0 {
                return Err(FormatError::InvalidData(
                    "fill-value message sets both the undefined and have-value flags".into(),
                ));
            }
            (0, None)
        } else if flags & FLAG_HAVE_VALUE != 0 {
            if buf.len() < pos + 4 {
                return Err(FormatError::BufferTooShort {
                    needed: pos + 4,
                    available: buf.len(),
                });
            }
            let size =
                u32::from_le_bytes([buf[pos], buf[pos + 1], buf[pos + 2], buf[pos + 3]]) as usize;
            pos += 4;
            if buf.len() < pos + size {
                return Err(FormatError::BufferTooShort {
                    needed: pos + size,
                    available: buf.len(),
                });
            }
            let data = buf[pos..pos + size].to_vec();
            pos += size;
            (2, Some(data))
        } else {
            (1, None)
        };

        Ok((
            Self {
                alloc_time,
                fill_write_time,
                fill_defined,
                fill_value,
            },
            pos,
        ))
    }
}

// ================================================================ tiling helper

/// Build a `total`-byte buffer whose contents are `fill_value` tiled one
/// element wide, or all zeros when `fill_value` is `None` or empty.
///
/// This is the single source of truth for materializing fill values:
/// chunked reads use it to initialize output buffers, and the writer uses
/// it to pad partial chunks so that unwritten elements read back as the
/// fill value rather than zero.
pub(crate) fn tiled_fill(total: usize, fill_value: Option<&[u8]>) -> Vec<u8> {
    match fill_value {
        Some(fv) if !fv.is_empty() && total > 0 => {
            let mut buf = vec![0u8; total];
            for slot in buf.chunks_mut(fv.len()) {
                let n = slot.len().min(fv.len());
                slot[..n].copy_from_slice(&fv[..n]);
            }
            buf
        }
        _ => vec![0u8; total],
    }
}

/// Fallible variant of [`tiled_fill`] for reader paths.
///
/// `total` on a read path is derived from untrusted file fields (dataspace
/// dimensions, element size). A crafted file can declare an absurd dataset
/// size; allocating it with `vec![0u8; total]` aborts the process on
/// allocation failure. This variant uses `try_reserve_exact`, returning a
/// `TryReserveError` the caller can surface as a clean error instead.
pub(crate) fn try_tiled_fill(
    total: usize,
    fill_value: Option<&[u8]>,
) -> Result<Vec<u8>, std::collections::TryReserveError> {
    let mut buf: Vec<u8> = Vec::new();
    buf.try_reserve_exact(total)?;
    buf.resize(total, 0);
    if let Some(fv) = fill_value {
        if !fv.is_empty() && total > 0 {
            for slot in buf.chunks_mut(fv.len()) {
                let n = slot.len().min(fv.len());
                slot[..n].copy_from_slice(&fv[..n]);
            }
        }
    }
    Ok(buf)
}

// ======================================================================= tests

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn roundtrip_default() {
        let msg = FillValueMessage::default();
        let encoded = msg.encode();
        // Version 3: version byte + flags byte, no value.
        assert_eq!(encoded.len(), 2);
        let (decoded, consumed) = FillValueMessage::decode(&encoded).unwrap();
        assert_eq!(consumed, 2);
        assert_eq!(decoded, msg);
    }

    #[test]
    fn roundtrip_user_defined() {
        let msg = FillValueMessage::with_value(vec![0xDE, 0xAD, 0xBE, 0xEF]);
        let encoded = msg.encode();
        // version + flags + u32 size + 4 data = 10
        assert_eq!(encoded.len(), 10);
        let (decoded, consumed) = FillValueMessage::decode(&encoded).unwrap();
        assert_eq!(consumed, 10);
        assert_eq!(decoded, msg);
        assert_eq!(
            decoded.fill_value.as_ref().unwrap(),
            &vec![0xDE, 0xAD, 0xBE, 0xEF]
        );
    }

    #[test]
    fn roundtrip_undefined() {
        let msg = FillValueMessage::undefined();
        let encoded = msg.encode();
        assert_eq!(encoded.len(), 2);
        let (decoded, consumed) = FillValueMessage::decode(&encoded).unwrap();
        assert_eq!(consumed, 2);
        assert_eq!(decoded, msg);
    }

    #[test]
    fn version_3_flags_byte_layout() {
        // alloc_time=3 (bits 0-1), fill_write_time=2 (bits 2-3),
        // explicit value present -> 0x20.
        let msg = FillValueMessage {
            alloc_time: 3,
            fill_write_time: 2,
            fill_defined: 2,
            fill_value: Some(vec![0x01, 0x02]),
        };
        let encoded = msg.encode();
        assert_eq!(encoded[0], 3);
        assert_eq!(encoded[1], 0x03 | (0x02 << 2) | FLAG_HAVE_VALUE);
        assert_eq!(&encoded[2..6], &2u32.to_le_bytes());
        assert_eq!(&encoded[6..8], &[0x01, 0x02]);
        assert_eq!(encoded.len(), 8);
    }

    #[test]
    fn version_3_undefined_flag() {
        let encoded = FillValueMessage::undefined().encode();
        assert_eq!(encoded[1] & FLAG_UNDEFINED, FLAG_UNDEFINED);
        assert_eq!(encoded[1] & FLAG_HAVE_VALUE, 0);
    }

    #[test]
    fn empty_user_data_normalizes_to_default() {
        // An empty explicit value is indistinguishable from default fill in
        // the on-disk format, so it round-trips as `fill_defined == 1`.
        let msg = FillValueMessage {
            alloc_time: 1,
            fill_write_time: 2,
            fill_defined: 2,
            fill_value: Some(vec![]),
        };
        let encoded = msg.encode();
        assert_eq!(encoded.len(), 2);
        let (decoded, consumed) = FillValueMessage::decode(&encoded).unwrap();
        assert_eq!(consumed, 2);
        assert_eq!(decoded.alloc_time, 1);
        assert_eq!(decoded.fill_write_time, 2);
        assert_eq!(decoded.fill_defined, 1);
        assert_eq!(decoded.fill_value, None);
    }

    #[test]
    fn decode_version_1_message() {
        // Version-1 message with an explicit 4-byte value.
        let buf = [1u8, 2, 0, 1, 4, 0, 0, 0, 0xAA, 0xBB, 0xCC, 0xDD];
        let (decoded, consumed) = FillValueMessage::decode(&buf).unwrap();
        assert_eq!(consumed, 12);
        assert_eq!(decoded.alloc_time, 2);
        assert_eq!(decoded.fill_defined, 2);
        assert_eq!(
            decoded.fill_value.as_ref().unwrap(),
            &vec![0xAA, 0xBB, 0xCC, 0xDD]
        );
    }

    #[test]
    fn decode_version_2_message_libhdf5_default() {
        // The body libhdf5 2.0.0 actually writes: version 2, alloc=2,
        // fill_time=2, defined=1, size=4.
        let buf = [2u8, 2, 2, 1, 4, 0, 0, 0, 0x00, 0x00, 0x80, 0xBF];
        let (decoded, consumed) = FillValueMessage::decode(&buf).unwrap();
        assert_eq!(consumed, 12);
        assert_eq!(decoded.alloc_time, 2);
        assert_eq!(decoded.fill_write_time, 2);
        assert_eq!(decoded.fill_defined, 2);
        assert_eq!(
            decoded.fill_value.as_ref().unwrap(),
            &vec![0x00, 0x00, 0x80, 0xBF]
        );
    }

    #[test]
    fn decode_version_2_defined_without_value() {
        // Defined (byte != 0) but size 0: no explicit value.
        let buf = [2u8, 2, 0, 1, 0, 0, 0, 0];
        let (decoded, consumed) = FillValueMessage::decode(&buf).unwrap();
        assert_eq!(consumed, 8);
        assert_eq!(decoded.fill_defined, 1);
        assert_eq!(decoded.fill_value, None);
    }

    #[test]
    fn decode_bad_version() {
        for bad in [0u8, 4, 9] {
            let buf = [bad, 0, 0, 0];
            match FillValueMessage::decode(&buf).unwrap_err() {
                FormatError::InvalidVersion(v) if v == bad => {}
                other => panic!("unexpected error for version {bad}: {other:?}"),
            }
        }
    }

    #[test]
    fn decode_buffer_too_short() {
        // Only the version byte — a v3 message needs the flags byte too.
        let buf = [3u8];
        match FillValueMessage::decode(&buf).unwrap_err() {
            FormatError::BufferTooShort { .. } => {}
            other => panic!("unexpected error: {other:?}"),
        }
    }

    #[test]
    fn decode_v3_unknown_flag_rejected() {
        // Bit 0x40 is not a defined flag.
        let buf = [3u8, 0x40];
        match FillValueMessage::decode(&buf).unwrap_err() {
            FormatError::InvalidData(_) => {}
            other => panic!("unexpected error: {other:?}"),
        }
    }

    #[test]
    fn decode_v3_truncated_size() {
        // HAVE_VALUE flag set but the u32 size field is missing.
        let buf = [3u8, FLAG_HAVE_VALUE, 0xFF];
        match FillValueMessage::decode(&buf).unwrap_err() {
            FormatError::BufferTooShort { .. } => {}
            other => panic!("unexpected error: {other:?}"),
        }
    }

    #[test]
    fn decode_v3_truncated_data() {
        // HAVE_VALUE, size=4, but only 2 bytes of data.
        let buf = [3u8, FLAG_HAVE_VALUE, 4, 0, 0, 0, 0xAA, 0xBB];
        match FillValueMessage::decode(&buf).unwrap_err() {
            FormatError::BufferTooShort {
                needed: 10,
                available: 8,
            } => {}
            other => panic!("unexpected error: {other:?}"),
        }
    }

    #[test]
    fn version_byte() {
        let encoded = FillValueMessage::default().encode();
        assert_eq!(encoded[0], 3);
    }

    #[test]
    fn tiled_fill_repeats_pattern() {
        assert_eq!(tiled_fill(0, Some(&[1, 2])), Vec::<u8>::new());
        assert_eq!(tiled_fill(6, None), vec![0u8; 6]);
        assert_eq!(tiled_fill(6, Some(&[])), vec![0u8; 6]);
        assert_eq!(
            tiled_fill(6, Some(&[0xAB, 0xCD])),
            vec![0xAB, 0xCD, 0xAB, 0xCD, 0xAB, 0xCD]
        );
        // Partial tail when total is not a multiple of the pattern width.
        assert_eq!(tiled_fill(5, Some(&[1, 2])), vec![1, 2, 1, 2, 1]);
    }
}