hdf5-pure 0.28.0

//! Builder types for HDF5 datatypes, attributes, datasets, and groups.
//!
//! Extracted from `file_writer.rs` to keep modules under the line limit.

#[cfg(not(feature = "std"))]
use alloc::{boxed::Box, string::String, string::ToString, vec, vec::Vec};

use crate::attribute::AttributeMessage;
use crate::chunked_write::{ChunkMeta, ChunkOptions, ChunkProvider};
use crate::compound::CompoundType;
use crate::convert::TryToUsize;
use crate::dataspace::{Dataspace, DataspaceType};
use crate::datatype::{
    CharacterSet, CompoundMember, Datatype, DatatypeByteOrder, EnumMember, StringPadding,
};
use crate::error::FormatError;
use crate::scaleoffset::ScaleOffset;

// ---- Datatype constructors ----

pub fn make_f64_type() -> Datatype {
    Datatype::FloatingPoint {
        size: 8,
        byte_order: DatatypeByteOrder::LittleEndian,
        bit_offset: 0,
        bit_precision: 64,
        exponent_location: 52,
        exponent_size: 11,
        mantissa_location: 0,
        mantissa_size: 52,
        exponent_bias: 1023,
    }
}

pub fn make_f32_type() -> Datatype {
    Datatype::FloatingPoint {
        size: 4,
        byte_order: DatatypeByteOrder::LittleEndian,
        bit_offset: 0,
        bit_precision: 32,
        exponent_location: 23,
        exponent_size: 8,
        mantissa_location: 0,
        mantissa_size: 23,
        exponent_bias: 127,
    }
}

pub fn make_i32_type() -> Datatype {
    Datatype::FixedPoint {
        size: 4,
        byte_order: DatatypeByteOrder::LittleEndian,
        signed: true,
        bit_offset: 0,
        bit_precision: 32,
    }
}

pub fn make_i64_type() -> Datatype {
    Datatype::FixedPoint {
        size: 8,
        byte_order: DatatypeByteOrder::LittleEndian,
        signed: true,
        bit_offset: 0,
        bit_precision: 64,
    }
}

pub fn make_u8_type() -> Datatype {
    Datatype::FixedPoint {
        size: 1,
        byte_order: DatatypeByteOrder::LittleEndian,
        signed: false,
        bit_offset: 0,
        bit_precision: 8,
    }
}

pub fn make_i8_type() -> Datatype {
    Datatype::FixedPoint {
        size: 1,
        byte_order: DatatypeByteOrder::LittleEndian,
        signed: true,
        bit_offset: 0,
        bit_precision: 8,
    }
}

pub fn make_i16_type() -> Datatype {
    Datatype::FixedPoint {
        size: 2,
        byte_order: DatatypeByteOrder::LittleEndian,
        signed: true,
        bit_offset: 0,
        bit_precision: 16,
    }
}

pub fn make_u16_type() -> Datatype {
    Datatype::FixedPoint {
        size: 2,
        byte_order: DatatypeByteOrder::LittleEndian,
        signed: false,
        bit_offset: 0,
        bit_precision: 16,
    }
}

pub fn make_u32_type() -> Datatype {
    Datatype::FixedPoint {
        size: 4,
        byte_order: DatatypeByteOrder::LittleEndian,
        signed: false,
        bit_offset: 0,
        bit_precision: 32,
    }
}

pub fn make_u64_type() -> Datatype {
    Datatype::FixedPoint {
        size: 8,
        byte_order: DatatypeByteOrder::LittleEndian,
        signed: false,
        bit_offset: 0,
        bit_precision: 64,
    }
}

pub fn make_object_reference_type() -> Datatype {
    Datatype::Reference {
        size: 8,
        ref_type: crate::datatype::ReferenceType::Object,
    }
}

/// A variable-length string datatype with the given character set and
/// null-terminated padding.
///
/// The character set and padding live in the variable-length datatype's own
/// bitfields; the base element type is an 8-bit unsigned integer
/// (`H5T_STD_U8LE`), exactly the shape the reference C library and h5py emit for
/// a VL string (`H5Tvlen_create(H5T_C_S1)` stores the base as a 1-byte
/// integer). Matching it byte-for-byte is what lets the C library read these
/// datasets back into `VarLenUnicode`/`VarLenAscii` without a conversion-path
/// error.
pub fn make_vlen_string_type(charset: CharacterSet) -> Datatype {
    Datatype::VariableLength {
        is_string: true,
        padding: Some(StringPadding::NullTerminate),
        charset: Some(charset),
        base_type: Box::new(make_u8_type()),
    }
}

// ---- Compound / Enum type builders ----

/// Builder for constructing HDF5 compound (struct) datatypes.
pub struct CompoundTypeBuilder {
    fields: Vec<(String, Datatype)>,
}

impl CompoundTypeBuilder {
    pub fn new() -> Self {
        Self { fields: Vec::new() }
    }

    /// Add a named field with the given datatype.
    pub fn field(mut self, name: &str, datatype: Datatype) -> Self {
        self.fields.push((name.to_string(), datatype));
        self
    }

    /// Add an f64 field.
    pub fn f64_field(self, name: &str) -> Self {
        self.field(name, make_f64_type())
    }
    /// Add an f32 field.
    pub fn f32_field(self, name: &str) -> Self {
        self.field(name, make_f32_type())
    }
    /// Add an i32 field.
    pub fn i32_field(self, name: &str) -> Self {
        self.field(name, make_i32_type())
    }
    /// Add an i64 field.
    pub fn i64_field(self, name: &str) -> Self {
        self.field(name, make_i64_type())
    }
    /// Add a u8 field.
    pub fn u8_field(self, name: &str) -> Self {
        self.field(name, make_u8_type())
    }
    /// Add an i8 field.
    pub fn i8_field(self, name: &str) -> Self {
        self.field(name, make_i8_type())
    }
    /// Add an i16 field.
    pub fn i16_field(self, name: &str) -> Self {
        self.field(name, make_i16_type())
    }
    /// Add a u16 field.
    pub fn u16_field(self, name: &str) -> Self {
        self.field(name, make_u16_type())
    }
    /// Add a u32 field.
    pub fn u32_field(self, name: &str) -> Self {
        self.field(name, make_u32_type())
    }
    /// Add a u64 field.
    pub fn u64_field(self, name: &str) -> Self {
        self.field(name, make_u64_type())
    }

    /// Build the compound datatype.
    pub fn build(self) -> Datatype {
        let mut offset = 0u64;
        let mut members = Vec::with_capacity(self.fields.len());
        for (name, dt) in self.fields {
            let sz = dt.type_size();
            members.push(CompoundMember {
                name,
                byte_offset: offset,
                datatype: dt,
            });
            offset += sz as u64;
        }
        Datatype::Compound {
            #[expect(
                clippy::cast_possible_truncation,
                reason = "accumulated compound size is stored in the 4-byte datatype size field"
            )]
            size: offset as u32,
            members,
        }
    }
}

impl Default for CompoundTypeBuilder {
    fn default() -> Self {
        Self::new()
    }
}

mod complex_component {
    pub trait Sealed {}
}

/// A scalar type that can be the component of a complex `{real, imag}` dataset.
///
/// A complex array is a two-field compound of one numeric class, so the only
/// things the layout depends on are the component's datatype and its
/// little-endian encoding — which is exactly what this trait supplies, and why
/// [`DatasetBuilder::with_complex_data`] needs nothing per width beyond a new
/// impl here.
///
/// Sealed: the impls below cover every class the crate can store, and a
/// component type outside that set could only produce a malformed file.
pub(crate) trait ComplexComponent: complex_component::Sealed + Copy {
    /// The datatype of one component, identical to what the type's
    /// `with_*_data` writer emits for a real dataset of the same class.
    fn datatype() -> Datatype;

    /// Append `self` to `out` in little-endian byte order.
    fn encode_le(self, out: &mut Vec<u8>);

    /// Decode one component from exactly `size_of::<Self>()` little-endian
    /// bytes. Callers slice the element out of the raw buffer first, so a
    /// wrong length is a bug here rather than bad input.
    ///
    /// Gated to match its only caller: the MAT reader is `serde`-only, while
    /// the writer half of this trait compiles unconditionally.
    #[cfg(feature = "serde")]
    fn decode_le(bytes: &[u8]) -> Self;
}

macro_rules! impl_complex_component {
    ($($ty:ty => $make:ident),* $(,)?) => {
        $(
            impl complex_component::Sealed for $ty {}
            impl ComplexComponent for $ty {
                fn datatype() -> Datatype {
                    $make()
                }
                fn encode_le(self, out: &mut Vec<u8>) {
                    out.extend_from_slice(&self.to_le_bytes());
                }
                #[cfg(feature = "serde")]
                fn decode_le(bytes: &[u8]) -> Self {
                    Self::from_le_bytes(
                        bytes
                            .try_into()
                            .expect("caller slices exactly one component"),
                    )
                }
            }
        )*
    };
}

impl_complex_component! {
    f64 => make_f64_type,
    f32 => make_f32_type,
    i64 => make_i64_type,
    i32 => make_i32_type,
    i16 => make_i16_type,
    i8 => make_i8_type,
    u64 => make_u64_type,
    u32 => make_u32_type,
    u16 => make_u16_type,
    u8 => make_u8_type,
}

/// Builder for an HDF5 compound datatype with explicit field offsets and size.
///
/// This is the pure-Rust equivalent of creating an `H5T_COMPOUND` type and
/// inserting fields with `H5Tinsert`. [`build`](Self::build) validates field
/// names, bounds, and overlap before returning a datatype.
pub struct ExplicitCompoundTypeBuilder {
    size: u32,
    fields: Vec<CompoundMember>,
}

impl ExplicitCompoundTypeBuilder {
    /// Add a field at an explicit byte offset.
    pub fn field(mut self, name: &str, byte_offset: u64, datatype: Datatype) -> Self {
        self.fields.push(CompoundMember {
            name: name.to_string(),
            byte_offset,
            datatype,
        });
        self
    }

    /// Add an f64 field at an explicit byte offset.
    pub fn f64_field(self, name: &str, byte_offset: u64) -> Self {
        self.field(name, byte_offset, make_f64_type())
    }

    /// Add an f32 field at an explicit byte offset.
    pub fn f32_field(self, name: &str, byte_offset: u64) -> Self {
        self.field(name, byte_offset, make_f32_type())
    }

    /// Add an i32 field at an explicit byte offset.
    pub fn i32_field(self, name: &str, byte_offset: u64) -> Self {
        self.field(name, byte_offset, make_i32_type())
    }

    /// Add an i64 field at an explicit byte offset.
    pub fn i64_field(self, name: &str, byte_offset: u64) -> Self {
        self.field(name, byte_offset, make_i64_type())
    }

    /// Add a u8 field at an explicit byte offset.
    pub fn u8_field(self, name: &str, byte_offset: u64) -> Self {
        self.field(name, byte_offset, make_u8_type())
    }

    /// Add an i8 field at an explicit byte offset.
    pub fn i8_field(self, name: &str, byte_offset: u64) -> Self {
        self.field(name, byte_offset, make_i8_type())
    }

    /// Add an i16 field at an explicit byte offset.
    pub fn i16_field(self, name: &str, byte_offset: u64) -> Self {
        self.field(name, byte_offset, make_i16_type())
    }

    /// Add a u16 field at an explicit byte offset.
    pub fn u16_field(self, name: &str, byte_offset: u64) -> Self {
        self.field(name, byte_offset, make_u16_type())
    }

    /// Add a u32 field at an explicit byte offset.
    pub fn u32_field(self, name: &str, byte_offset: u64) -> Self {
        self.field(name, byte_offset, make_u32_type())
    }

    /// Add a u64 field at an explicit byte offset.
    pub fn u64_field(self, name: &str, byte_offset: u64) -> Self {
        self.field(name, byte_offset, make_u64_type())
    }

    /// Validate and build the compound datatype.
    pub fn build(mut self) -> Result<Datatype, crate::error::FormatError> {
        use crate::error::FormatError;

        if self.size == 0 {
            return Err(FormatError::InvalidCompoundSize);
        }
        if self.fields.is_empty() {
            return Err(FormatError::EmptyCompoundType);
        }

        for (index, field) in self.fields.iter().enumerate() {
            if self.fields[..index]
                .iter()
                .any(|earlier| earlier.name == field.name)
            {
                return Err(FormatError::DuplicateCompoundField(field.name.clone()));
            }
            let field_size = field.datatype.type_size();
            let end = field.byte_offset.checked_add(u64::from(field_size));
            if field_size == 0 || end.is_none_or(|end| end > u64::from(self.size)) {
                return Err(FormatError::CompoundFieldOutOfBounds {
                    name: field.name.clone(),
                    offset: field.byte_offset,
                    field_size,
                    compound_size: self.size,
                });
            }
        }

        self.fields.sort_by_key(|field| field.byte_offset);
        for fields in self.fields.windows(2) {
            let first_end = fields[0].byte_offset + u64::from(fields[0].datatype.type_size());
            if first_end > fields[1].byte_offset {
                return Err(FormatError::CompoundFieldOverlap {
                    first: fields[0].name.clone(),
                    second: fields[1].name.clone(),
                });
            }
        }

        Ok(Datatype::Compound {
            size: self.size,
            members: self.fields,
        })
    }
}

impl CompoundTypeBuilder {
    /// Create a compound builder with an explicit total size and field offsets.
    pub fn with_size(size: u32) -> ExplicitCompoundTypeBuilder {
        ExplicitCompoundTypeBuilder {
            size,
            fields: Vec::new(),
        }
    }
}

/// Builder for constructing HDF5 enumeration datatypes.
pub struct EnumTypeBuilder {
    base_type: Datatype,
    members: Vec<(String, PendingEnumValue)>,
}

/// A member value awaiting the base type's width, resolved by
/// [`EnumTypeBuilder::build`].
enum PendingEnumValue {
    /// An integer, encoded little-endian into the base type's size at build.
    Int(i64),
    /// Raw little-endian bytes, which must already match the base type's size.
    Raw(Vec<u8>),
}

impl EnumTypeBuilder {
    /// Create a new enum builder with i32 base type.
    pub fn i32_based() -> Self {
        Self::with_base(make_i32_type())
    }

    /// Create a new enum builder with u8 base type.
    pub fn u8_based() -> Self {
        Self::with_base(make_u8_type())
    }

    /// Create an enum builder over an arbitrary integer base type, such as
    /// [`make_u16_type`] or [`make_i64_type`].
    ///
    /// The enumeration's element size comes from `base_type`. A non-integer base
    /// is refused by [`build`](Self::build) with
    /// [`FormatError::EnumBaseNotInteger`].
    pub fn with_base(base_type: Datatype) -> Self {
        Self {
            base_type,
            members: Vec::new(),
        }
    }

    /// Add a named value, encoded little-endian into the base type's width.
    ///
    /// A value that does not fit the base type is refused by
    /// [`build`](Self::build) with [`FormatError::EnumMemberValueRange`].
    pub fn value(mut self, name: &str, val: i32) -> Self {
        self.members
            .push((name.to_string(), PendingEnumValue::Int(val as i64)));
        self
    }

    /// Add a named u8 value.
    pub fn u8_value(self, name: &str, val: u8) -> Self {
        self.value(name, i32::from(val))
    }

    /// Add a named value from an integer wide enough for any base type.
    pub fn i64_value(mut self, name: &str, val: i64) -> Self {
        self.members
            .push((name.to_string(), PendingEnumValue::Int(val)));
        self
    }

    /// Add a named value from its raw little-endian bytes — the form the format
    /// stores and [`EnumMember::value`] holds.
    ///
    /// `bytes` must be exactly the base type's size, or [`build`](Self::build)
    /// refuses it with [`FormatError::EnumMemberValueSize`]. Use this for a base
    /// type whose values do not fit an `i64`, or to reproduce stored bytes
    /// verbatim.
    pub fn raw_value(mut self, name: &str, bytes: &[u8]) -> Self {
        self.members
            .push((name.to_string(), PendingEnumValue::Raw(bytes.to_vec())));
        self
    }

    /// Build the enumeration datatype, resolving every member against the base
    /// type's width.
    ///
    /// Fails if the base type is not an integer, if a member's raw byte length
    /// disagrees with the base type's size, or if a member's integer value does
    /// not fit — rather than emitting a datatype message the reference C library
    /// cannot read.
    pub fn build(self) -> Result<Datatype, FormatError> {
        let size = self.base_type.type_size();
        let signed = match &self.base_type {
            Datatype::FixedPoint { signed, .. } => *signed,
            _ => return Err(FormatError::EnumBaseNotInteger),
        };
        let width = size.to_usize()?;

        let mut members = Vec::with_capacity(self.members.len());
        for (name, pending) in self.members {
            let value = match pending {
                PendingEnumValue::Raw(bytes) => {
                    if bytes.len() != width {
                        return Err(FormatError::EnumMemberValueSize(name, size, bytes.len()));
                    }
                    bytes
                }
                PendingEnumValue::Int(v) => {
                    if !int_fits(v, width, signed) {
                        return Err(FormatError::EnumMemberValueRange(name, v, size));
                    }
                    v.to_le_bytes()[..width].to_vec()
                }
            };
            members.push(EnumMember { name, value });
        }

        Ok(Datatype::Enumeration {
            size,
            base_type: Box::new(self.base_type),
            members,
        })
    }
}

/// Whether `v` is representable in `width` bytes under `signed`.
fn int_fits(v: i64, width: usize, signed: bool) -> bool {
    if width == 0 {
        return false;
    }
    if width >= 8 {
        // Every `i64` fits eight signed bytes; an unsigned eight-byte base needs
        // a non-negative value (larger magnitudes need `raw_value`).
        return signed || v >= 0;
    }
    let bits = width * 8;
    if signed {
        let min = -(1i64 << (bits - 1));
        let max = (1i64 << (bits - 1)) - 1;
        (min..=max).contains(&v)
    } else {
        let max = (1i64 << bits) - 1;
        (0..=max).contains(&v)
    }
}

// ---- Attribute helper ----

pub(crate) fn build_attr_message(name: &str, value: &AttrValue) -> AttributeMessage {
    match value {
        AttrValue::F64(v) => AttributeMessage {
            name: name.to_string(),
            datatype: make_f64_type(),
            dataspace: scalar_ds(),
            raw_data: v.to_le_bytes().to_vec(),
        },
        AttrValue::F64Array(arr) => {
            let mut raw = Vec::with_capacity(arr.len() * 8);
            for v in arr {
                raw.extend_from_slice(&v.to_le_bytes());
            }
            AttributeMessage {
                name: name.to_string(),
                datatype: make_f64_type(),
                dataspace: simple_1d(arr.len() as u64),
                raw_data: raw,
            }
        }
        AttrValue::I64(v) => AttributeMessage {
            name: name.to_string(),
            datatype: make_i64_type(),
            dataspace: scalar_ds(),
            raw_data: v.to_le_bytes().to_vec(),
        },
        AttrValue::I64Array(arr) => {
            let mut raw = Vec::with_capacity(arr.len() * 8);
            for v in arr {
                raw.extend_from_slice(&v.to_le_bytes());
            }
            AttributeMessage {
                name: name.to_string(),
                datatype: make_i64_type(),
                dataspace: simple_1d(arr.len() as u64),
                raw_data: raw,
            }
        }
        AttrValue::I32(v) => AttributeMessage {
            name: name.to_string(),
            datatype: make_i32_type(),
            dataspace: scalar_ds(),
            raw_data: v.to_le_bytes().to_vec(),
        },
        AttrValue::U32(v) => AttributeMessage {
            name: name.to_string(),
            datatype: make_u32_type(),
            dataspace: scalar_ds(),
            raw_data: v.to_le_bytes().to_vec(),
        },
        AttrValue::U64(v) => AttributeMessage {
            name: name.to_string(),
            datatype: make_u64_type(),
            dataspace: scalar_ds(),
            raw_data: v.to_le_bytes().to_vec(),
        },
        AttrValue::String(s) => {
            let bytes = s.as_bytes();
            AttributeMessage {
                name: name.to_string(),
                datatype: Datatype::String {
                    #[expect(
                        clippy::cast_possible_truncation,
                        reason = "string byte length is stored in the 4-byte fixed-string datatype size field"
                    )]
                    size: bytes.len() as u32,
                    padding: StringPadding::NullPad,
                    charset: CharacterSet::Utf8,
                },
                dataspace: scalar_ds(),
                raw_data: bytes.to_vec(),
            }
        }
        AttrValue::StringArray(arr) => {
            let max_len = arr.iter().map(|s| s.len()).max().unwrap_or(0);
            let mut raw = Vec::new();
            for s in arr {
                let mut b = s.as_bytes().to_vec();
                b.resize(max_len, 0);
                raw.extend_from_slice(&b);
            }
            AttributeMessage {
                name: name.to_string(),
                datatype: Datatype::String {
                    #[expect(
                        clippy::cast_possible_truncation,
                        reason = "max string byte length is stored in the 4-byte fixed-string datatype size field"
                    )]
                    size: max_len as u32,
                    padding: StringPadding::NullPad,
                    charset: CharacterSet::Utf8,
                },
                dataspace: simple_1d(arr.len() as u64),
                raw_data: raw,
            }
        }
        AttrValue::AsciiString(s) => {
            let bytes = s.as_bytes();
            AttributeMessage {
                name: name.to_string(),
                datatype: Datatype::String {
                    #[expect(
                        clippy::cast_possible_truncation,
                        reason = "string byte length is stored in the 4-byte fixed-string datatype size field"
                    )]
                    size: bytes.len() as u32,
                    padding: StringPadding::NullPad,
                    charset: CharacterSet::Ascii,
                },
                dataspace: scalar_ds(),
                raw_data: bytes.to_vec(),
            }
        }
        AttrValue::AsciiStringArray(arr) => {
            let max_len = arr.iter().map(|s| s.len()).max().unwrap_or(0);
            let mut raw = Vec::new();
            for s in arr {
                let mut b = s.as_bytes().to_vec();
                b.resize(max_len, 0);
                raw.extend_from_slice(&b);
            }
            AttributeMessage {
                name: name.to_string(),
                datatype: Datatype::String {
                    #[expect(
                        clippy::cast_possible_truncation,
                        reason = "max string byte length is stored in the 4-byte fixed-string datatype size field"
                    )]
                    size: max_len as u32,
                    padding: StringPadding::NullPad,
                    charset: CharacterSet::Ascii,
                },
                dataspace: simple_1d(arr.len() as u64),
                raw_data: raw,
            }
        }
        AttrValue::VarLenAsciiArray(strings) => {
            // MATLAB v7.3 (and matio) expect MATLAB_fields and similar
            // variable-length ASCII arrays encoded as:
            //   H5T_VLEN { H5T_STRING { STRSIZE=1, NULLTERM, ASCII } }
            // — a VLEN sequence of 1-byte fixed strings. The on-disk byte
            // layout is identical to H5T_STRING{STRSIZE=VAR} (length + heap
            // address + object index per element; heap object holds raw
            // bytes without null terminator), so only the datatype
            // descriptor changes.
            let vl_ref_size = 16usize; // 4 + 8 + 4 for offset_size=8
            let mut raw = Vec::with_capacity(strings.len() * vl_ref_size);
            for (i, s) in strings.iter().enumerate() {
                #[expect(
                    clippy::cast_possible_truncation,
                    reason = "VLEN string length is written into the 4-byte length prefix of the variable-length reference"
                )]
                raw.extend_from_slice(&(s.len() as u32).to_le_bytes());
                raw.extend_from_slice(&0u64.to_le_bytes()); // patched later
                // 1-based index within this element's own collection; the
                // writer splits the strings across collections the same way
                // (see `build_global_heap_collections`).
                #[expect(
                    clippy::cast_possible_truncation,
                    reason = "1-based heap object index is written into the 4-byte object-index field of the variable-length reference"
                )]
                raw.extend_from_slice(&((i % MAX_HEAP_OBJECTS + 1) as u32).to_le_bytes());
            }
            AttributeMessage {
                name: name.to_string(),
                datatype: Datatype::VariableLength {
                    is_string: false,
                    padding: None,
                    charset: None,
                    base_type: Box::new(Datatype::String {
                        size: 1,
                        padding: StringPadding::NullTerminate,
                        charset: CharacterSet::Ascii,
                    }),
                },
                dataspace: simple_1d(strings.len() as u64),
                raw_data: raw,
            }
        }
    }
}

/// Maximum number of objects one global heap collection can index.
///
/// The heap-object index field is a `u16` with 0 reserved for the free-space
/// marker, so a single collection addresses at most `u16::MAX` objects. Data
/// with more objects than this is split across consecutive collections, whose
/// indices restart at 1 — the same thing the reference C library does when a
/// collection fills.
pub(crate) const MAX_HEAP_OBJECTS: usize = u16::MAX as usize;

/// Build the global heap collections holding the given strings, splitting them
/// across as many collections as their count needs.
pub(crate) fn build_global_heap_collections(strings: &[&str]) -> Vec<Vec<u8>> {
    let objects: Vec<&[u8]> = strings.iter().map(|s| s.as_bytes()).collect();
    build_global_heap_collections_from_bytes(&objects)
}

/// Build the global heap collections holding the given raw byte objects (no
/// UTF-8 requirement), in order. Mirrors [`build_global_heap_collections`] but
/// accepts arbitrary bytes so a faithful rewrite can carry embedded-NUL or
/// non-UTF-8 VL payloads.
///
/// Objects are packed [`MAX_HEAP_OBJECTS`] to a collection, so object `n` lives
/// in collection `n / MAX_HEAP_OBJECTS` at 1-based index
/// `n % MAX_HEAP_OBJECTS + 1`. [`patch_vl_refs`] and [`patch_vl_refs_masked`]
/// resolve references with that same rule, and [`stage_vl_elements`] and
/// [`build_attr_message`] write the matching indices.
pub(crate) fn build_global_heap_collections_from_bytes(objects: &[&[u8]]) -> Vec<Vec<u8>> {
    objects
        .chunks(MAX_HEAP_OBJECTS)
        .map(build_global_heap_collection_bytes)
        .collect()
}

/// Build one global heap collection holding `objects` (at most
/// [`MAX_HEAP_OBJECTS`] of them), assigning 1-based object indices in order.
/// Returns the serialized collection bytes.
fn build_global_heap_collection_bytes(objects: &[&[u8]]) -> Vec<u8> {
    debug_assert!(
        objects.len() <= MAX_HEAP_OBJECTS,
        "a collection's 2-byte object index cannot address more than {MAX_HEAP_OBJECTS} objects"
    );
    let length_size = 8usize;
    let header_size = 8 + length_size; // sig(4) + ver(1) + reserved(3) + collection_size

    // Calculate total size
    let mut obj_size_total = 0usize;
    for obj in objects {
        let obj_header = 8 + length_size; // index(2) + refcount(2) + reserved(4) + size
        let padded_data_len = (obj.len() + 7) & !7; // pad to 8 bytes
        obj_size_total += obj_header + padded_data_len;
    }
    obj_size_total += 8 + length_size; // free space marker (full object header size)
    let collection_size = header_size + obj_size_total;
    // The C HDF5 library enforces a minimum collection size of 4096 bytes.
    let min_collection_size = 4096;
    let padded_collection = ((collection_size.max(min_collection_size)) + 7) & !7;

    let mut buf = Vec::with_capacity(padded_collection);
    // Header
    buf.extend_from_slice(b"GCOL");
    buf.push(1); // version
    buf.extend_from_slice(&[0u8; 3]); // reserved
    buf.extend_from_slice(&(padded_collection as u64).to_le_bytes());

    // Objects (1-based indices)
    for (i, obj) in objects.iter().enumerate() {
        #[expect(
            clippy::cast_possible_truncation,
            reason = "1-based heap object index is written into the 2-byte heap-object index field"
        )]
        let index = (i + 1) as u16;
        buf.extend_from_slice(&index.to_le_bytes());
        buf.extend_from_slice(&1u16.to_le_bytes()); // ref_count
        buf.extend_from_slice(&[0u8; 4]); // reserved
        buf.extend_from_slice(&(obj.len() as u64).to_le_bytes());
        buf.extend_from_slice(obj);
        // Pad to 8-byte boundary
        let padded = (obj.len() + 7) & !7;
        for _ in obj.len()..padded {
            buf.push(0);
        }
    }

    // Free space marker (index 0): the C library uses this size as the total
    // skip distance from the start of the object (including its header), so
    // it must equal the remaining bytes in the collection from this point.
    let free_total_size = padded_collection - buf.len();
    buf.extend_from_slice(&0u16.to_le_bytes()); // index 0
    buf.extend_from_slice(&0u16.to_le_bytes()); // ref_count
    buf.extend_from_slice(&[0u8; 4]); // reserved
    buf.extend_from_slice(&(free_total_size as u64).to_le_bytes()); // size

    // Pad collection to full size
    buf.resize(padded_collection, 0);

    buf
}

/// Patch VL attribute references with the actual global heap collection
/// addresses. The raw_data contains VL references with placeholder addresses
/// (0), one per attribute element, each holding an object of the collection its
/// position selects (see [`build_global_heap_collections_from_bytes`]);
/// `collection_addresses` gives those collections' placed addresses in order.
pub(crate) fn patch_vl_refs(raw_data: &mut [u8], collection_addresses: &[u64]) {
    let count = raw_data.len() / VL_REF_SIZE;
    for i in 0..count {
        let address = collection_addresses[i / MAX_HEAP_OBJECTS];
        let addr_offset = i * VL_REF_SIZE + 4; // skip sequence_length
        raw_data[addr_offset..addr_offset + 8].copy_from_slice(&address.to_le_bytes());
    }
}

/// A single element of a VL-string dataset being written: either a null
/// reference (no heap object) or a heap object carrying these exact bytes.
///
/// The two are distinct in the HDF5 model: a null reference reads back as a
/// null/empty element with no heap object, whereas a zero-length heap object
/// reads back as an empty string `""`. Carrying both lets a faithful rewrite
/// reproduce the source byte-for-byte.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum VlStringElement {
    /// A null reference: length 0, undefined heap address, no heap object.
    Null,
    /// A heap object holding these exact bytes (possibly empty).
    Bytes(Vec<u8>),
}

/// 16-byte size of a single VL global-heap reference (offset_size = 8):
/// length(4) + collection address(8) + object index(4).
pub(crate) const VL_REF_SIZE: usize = 16;

/// Staged variable-length dataset/attribute element data: the element bytes
/// (carrying VL references with placeholder heap addresses) plus the global heap
/// collections holding the non-null elements' bytes.
///
/// The non-null elements' bytes become heap objects in order, packed
/// [`MAX_HEAP_OBJECTS`] to a collection, and each reference carries its
/// object's 1-based index within its own collection — matching
/// [`build_global_heap_collections_from_bytes`]. Null elements carry a zero
/// address and object index 0, and are never patched so they read back as null.
pub(crate) struct VlStringStaging {
    /// The dataset/attribute element bytes. For a dataset whose datatype *is*
    /// variable-length this is one 16-byte reference per element; for one that
    /// merely *contains* variable-length members (a compound, or an array of
    /// them) it is the full element bytes with the embedded references stamped
    /// in place.
    pub refs: Vec<u8>,
    /// The serialized global heap collections holding the non-null objects, in
    /// the order their objects appear. Empty when there are no such objects.
    pub collections: Vec<Vec<u8>>,
    /// Byte offset within [`refs`](Self::refs) of each reference that names a
    /// heap object and so needs its address patched once the collections are
    /// placed, in the same order as those objects. Null references are absent:
    /// their address must stay zero so they read back as null.
    pub patch_offsets: Vec<usize>,
}

/// Stage the references and global-heap collections for a variable-length
/// dataset/attribute from its per-element byte payloads.
///
/// `element_size` is the byte width of the VL base type. A VL string's base type
/// is a single byte (`element_size == 1`), so the reference's stored `length`
/// equals the payload's byte count. A non-string VL sequence stores an element
/// *count* in that field, so the length written is `bytes.len() / element_size`,
/// while the heap object still holds the exact bytes.
pub(crate) fn stage_vl_elements(
    elements: &[VlStringElement],
    element_size: usize,
) -> VlStringStaging {
    let element_size = element_size.max(1);
    // Collect the non-null payloads in order; their positions become the heap
    // object indices, 1-based within each collection.
    let mut objects: Vec<&[u8]> = Vec::new();
    let mut refs = Vec::with_capacity(elements.len() * VL_REF_SIZE);
    let mut patch_offsets = Vec::with_capacity(elements.len());
    for element in elements {
        match element {
            VlStringElement::Null => {
                // A null VL reference: HDF5 marks "no heap object" with a zero
                // heap address (`H5T__vlen_disk_isnull` tests addr == 0), not the
                // all-ones "undefined address" sentinel — the reference C library
                // rejects the latter as a bad heap index when reading.
                refs.extend_from_slice(&0u32.to_le_bytes()); // length 0
                refs.extend_from_slice(&0u64.to_le_bytes()); // null heap address
                refs.extend_from_slice(&0u32.to_le_bytes()); // object index 0
            }
            VlStringElement::Bytes(bytes) => {
                patch_offsets.push(refs.len());
                #[expect(
                    clippy::cast_possible_truncation,
                    reason = "VL element length (element count) is written into the 4-byte \
                              length prefix of the variable-length reference"
                )]
                refs.extend_from_slice(&((bytes.len() / element_size) as u32).to_le_bytes());
                refs.extend_from_slice(&0u64.to_le_bytes()); // patched later
                // 1-based index within this object's own collection.
                let index = objects.len() % MAX_HEAP_OBJECTS + 1;
                #[expect(
                    clippy::cast_possible_truncation,
                    reason = "1-based heap object index is written into the 4-byte object-index \
                              field of the variable-length reference"
                )]
                refs.extend_from_slice(&(index as u32).to_le_bytes());
                objects.push(bytes);
            }
        }
    }
    // A dataset of only null elements (or an empty dataset) references no heap
    // object, so it gets no collection at all — there is nothing to patch and a
    // 4096-byte empty GCOL would be dead weight.
    VlStringStaging {
        refs,
        collections: build_global_heap_collections_from_bytes(&objects),
        patch_offsets,
    }
}

/// Stage the global-heap collections for a dataset whose datatype merely
/// *contains* variable-length members — a compound with a VL member, or an array
/// of such compounds — rather than being variable-length itself.
///
/// `raw` is the dataset's element bytes as read from the source, and `offsets`
/// gives the byte offset of every embedded VL reference within `raw`, in the same
/// order as `elements`. Each reference is rewritten in place: a null element is
/// zeroed outright, and a heap-backed one keeps the source's `length` field
/// (which counts base-type elements, whose width varies per member) while
/// gaining the destination's object index. The addresses stay at zero until
/// [`patch_vl_refs_masked`] fills them in, exactly as for a top-level VL dataset.
pub(crate) fn stage_embedded_vl_elements(
    mut raw: Vec<u8>,
    offsets: &[usize],
    elements: &[VlStringElement],
) -> VlStringStaging {
    debug_assert_eq!(
        offsets.len(),
        elements.len(),
        "one staged payload per embedded variable-length reference"
    );
    let mut objects: Vec<&[u8]> = Vec::new();
    let mut patch_offsets = Vec::with_capacity(elements.len());
    for (&offset, element) in offsets.iter().zip(elements) {
        let slot = &mut raw[offset..offset + VL_REF_SIZE];
        match element {
            VlStringElement::Null => slot.fill(0),
            VlStringElement::Bytes(bytes) => {
                patch_offsets.push(offset);
                // Leave the source's element-count `length` in bytes 0..4 alone,
                // and blank the address so an unpatched reference is visibly null
                // rather than a stale source address.
                slot[4..12].fill(0);
                let index = objects.len() % MAX_HEAP_OBJECTS + 1;
                #[expect(
                    clippy::cast_possible_truncation,
                    reason = "1-based heap object index is written into the 4-byte object-index \
                              field of the variable-length reference"
                )]
                slot[12..16].copy_from_slice(&(index as u32).to_le_bytes());
                objects.push(bytes);
            }
        }
    }
    VlStringStaging {
        collections: build_global_heap_collections_from_bytes(&objects),
        refs: raw,
        patch_offsets,
    }
}

/// Patch the heap address of each VL reference named by `patch_offsets`, leaving
/// null references (which are absent from that list) with their zero address so
/// they read back as null. Mirrors [`patch_vl_refs`], but because only
/// heap-backed references are listed, the collection a reference resolves to is
/// selected by its *object* position rather than its element position — and the
/// references need not sit at a fixed stride, which is what lets a compound with
/// variable-length members share this path. `collection_addresses` holds the
/// placed addresses of [`VlStringStaging::collections`], in order.
pub(crate) fn patch_vl_refs_masked(
    raw_data: &mut [u8],
    patch_offsets: &[usize],
    collection_addresses: &[u64],
) {
    for (object_ordinal, &offset) in patch_offsets.iter().enumerate() {
        let address = collection_addresses[object_ordinal / MAX_HEAP_OBJECTS];
        let addr_offset = offset + 4; // skip sequence_length
        raw_data[addr_offset..addr_offset + 8].copy_from_slice(&address.to_le_bytes());
    }
}

pub(crate) fn scalar_ds() -> Dataspace {
    Dataspace {
        space_type: DataspaceType::Scalar,
        rank: 0,
        dimensions: vec![],
        max_dimensions: None,
    }
}

pub(crate) fn simple_1d(n: u64) -> Dataspace {
    Dataspace {
        space_type: DataspaceType::Simple,
        rank: 1,
        dimensions: vec![n],
        max_dimensions: None,
    }
}

// ---- Attribute values ----

/// Convenient attribute values for the write API.
///
/// Non-exhaustive: variants are added as this crate supports more attribute
/// datatypes, so match a read-back value with a `_` arm. Constructing the
/// variants below is unaffected.
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub enum AttrValue {
    F64(f64),
    F64Array(Vec<f64>),
    I32(i32),
    I64(i64),
    I64Array(Vec<i64>),
    U32(u32),
    U64(u64),
    /// UTF-8 string attribute (null-padded).
    String(String),
    StringArray(Vec<String>),
    /// Fixed-width ASCII string attribute (charset = ASCII).
    AsciiString(String),
    /// Array of fixed-width ASCII strings (null-padded to the longest element).
    /// Compatible with MATLAB `MATLAB_fields` and matio.
    AsciiStringArray(Vec<String>),
    /// Array of variable-length ASCII strings (MATLAB_fields pattern).
    /// Each element is a variable-length sequence of ASCII bytes.
    /// Requires a global heap collection in the file.
    VarLenAsciiArray(Vec<String>),
}

// ---- Dataset builder ----

/// Configuration for SHINES provenance metadata.
#[cfg(feature = "provenance")]
#[derive(Debug, Clone)]
pub struct ProvenanceConfig {
    pub creator: String,
    pub timestamp: String,
    pub source: Option<String>,
}

/// Everything [`DatasetBuilder::with_raw_chunks_lazy`] needs to re-emit a chunked
/// dataset by copying its source chunks verbatim (no decode/re-encode): the
/// per-chunk sizes/masks (enough to plan the destination layout without reading
/// any bytes), a provider that yields each chunk's bytes on demand at write time,
/// the source filter-pipeline message, and the chunk geometry. Built by repack
/// from a source [`Dataset`].
pub(crate) struct RawChunkPayload {
    /// Logical chunk dimensions (rank entries, not the trailing element size).
    pub(crate) chunk_dims: Vec<u64>,
    /// Datatype element size in bytes.
    pub(crate) element_size: usize,
    /// Full uncompressed byte size of one chunk
    /// (`product(chunk_dims) * element_size`), identical for every chunk.
    pub(crate) raw_size: u64,
    /// The verbatim source `FilterPipeline` message bytes, if the source had one.
    pub(crate) pipeline_message: Option<Vec<u8>>,
    /// Per-chunk sizes + filter masks in dense row-major grid order, one per slot.
    pub(crate) meta: Vec<ChunkMeta>,
    /// Yields each chunk's compressed bytes on demand during the write, so no
    /// more than one chunk's bytes are resident. Owns its source (e.g. an
    /// `Arc<File>`), so it carries no borrowed lifetime.
    ///
    /// Wrapped in [`AssertUnwindSafe`](core::panic::AssertUnwindSafe) so the
    /// boxed trait object does not strip the `UnwindSafe`/`RefUnwindSafe`
    /// auto-traits from the public builder types that transitively hold it
    /// (removing an auto-trait impl is a semver break). The assertion is sound:
    /// the provider performs only immutable reads and leaves no broken state on
    /// a panic. `ChunkProvider: Send + Sync` keeps the other two auto-traits.
    pub(crate) provider: core::panic::AssertUnwindSafe<Box<dyn ChunkProvider>>,
}

/// One element of an object-reference dataset written through the builder.
///
/// A reference either names an object by path (resolved to that object's
/// destination address during serialization) or carries a raw address verbatim.
/// The raw form preserves a null reference (address 0) or an undefined reference
/// (`HADDR_UNDEF`, all-ones) exactly, which a faithful rewrite (repack) needs.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum ObjectRefTarget {
    /// Resolve to the destination address of the object at this path.
    Path(String),
    /// Write this exact 8-byte address (e.g. 0 for null, `u64::MAX` for
    /// undefined).
    Raw(u64),
}

/// One object-reference address to resolve during serialization, and where in
/// the dataset's element bytes it sits.
///
/// The offset is explicit rather than implied by a fixed stride so that a
/// reference embedded in a larger element — a compound member, or an array entry
/// — is rewritten in place alongside the fixed-size bytes around it. A dataset
/// whose datatype *is* an object reference simply has one patch every 8 bytes.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct ObjectRefPatch {
    /// Byte offset of the 8-byte address within the dataset's element bytes.
    pub byte_offset: usize,
    /// What to write there.
    pub target: ObjectRefTarget,
}

/// Write a resolved object-reference address into a dataset's element bytes.
///
/// The offset comes from the datatype's own layout and `raw` is sized from the
/// same datatype, so a slot that does not fit means the two disagree — a bug in
/// whoever staged them, not input this can correct. The debug assertion catches
/// that in the test suite; in release the write is skipped rather than allowed to
/// corrupt a neighbouring field. Note what a skip leaves behind depends on the
/// caller: placeholder zeros (a null reference) for a dataset staged by
/// [`DatasetBuilder::with_object_references`], but the *source's* address for one
/// staged by [`DatasetBuilder::with_embedded_object_references`], whose buffer is
/// the source's element bytes. Neither is reachable without the assertion firing
/// first.
pub(crate) fn write_reference_address(raw: &mut [u8], byte_offset: usize, address: u64) {
    debug_assert!(
        byte_offset + 8 <= raw.len(),
        "object-reference slot at {byte_offset} does not fit {} element bytes",
        raw.len()
    );
    if let Some(slot) = raw.get_mut(byte_offset..byte_offset + 8) {
        slot.copy_from_slice(&address.to_le_bytes());
    }
}

/// Builder for datasets.
pub struct DatasetBuilder {
    pub(crate) name: String,
    pub(crate) datatype: Option<Datatype>,
    pub(crate) shape: Option<Vec<u64>>,
    pub(crate) maxshape: Option<Vec<u64>>,
    pub(crate) data: Option<Vec<u8>>,
    pub(crate) attrs: Vec<(String, AttrValue)>,
    pub(crate) chunk_options: ChunkOptions,
    /// When set, this dataset's chunks are copied verbatim from a source file
    /// (repack's verbatim path): the already-compressed chunk bytes, the source
    /// filter-pipeline message, and the geometry needed to lay them out. This
    /// takes precedence over `data` / `chunk_options` for chunked storage.
    pub(crate) raw_chunks: Option<RawChunkPayload>,
    /// When set, this dataset is an object-reference dataset whose element
    /// addresses are resolved (per-element by path, or written raw) during file
    /// serialization once every object's destination address is known.
    pub(crate) reference_targets: Option<Vec<ObjectRefPatch>>,
    /// When set, this dataset stores variable-length strings: `data` holds the
    /// 16-byte references with placeholder heap addresses, and this staging
    /// carries the global heap collection plus the mask of references to patch
    /// once the post-data cursor is known.
    pub(crate) vl_string_staging: Option<VlStringStaging>,
    /// A user-defined fill value, encoded in the dataset's datatype (little-
    /// endian, one element wide). `None` leaves the crate's library-default fill
    /// value message untouched. Its byte width is checked against the datatype's
    /// element size when the dataset is serialized.
    pub(crate) fill: Option<Vec<u8>>,
    #[cfg(feature = "provenance")]
    pub(crate) provenance: Option<ProvenanceConfig>,
}

impl DatasetBuilder {
    pub(crate) fn new(name: &str) -> Self {
        Self {
            name: name.to_string(),
            datatype: None,
            shape: None,
            maxshape: None,
            data: None,
            attrs: Vec::new(),
            chunk_options: ChunkOptions::default(),
            raw_chunks: None,
            reference_targets: None,
            vl_string_staging: None,
            fill: None,
            #[cfg(feature = "provenance")]
            provenance: None,
        }
    }

    pub fn with_f64_data(&mut self, data: &[f64]) -> &mut Self {
        self.datatype = Some(make_f64_type());
        let mut b = Vec::with_capacity(data.len() * 8);
        for &v in data {
            b.extend_from_slice(&v.to_le_bytes());
        }
        self.data = Some(b);
        if self.shape.is_none() {
            self.shape = Some(vec![data.len() as u64]);
        }
        self
    }

    pub fn with_f32_data(&mut self, data: &[f32]) -> &mut Self {
        self.datatype = Some(make_f32_type());
        let mut b = Vec::with_capacity(data.len() * 4);
        for &v in data {
            b.extend_from_slice(&v.to_le_bytes());
        }
        self.data = Some(b);
        if self.shape.is_none() {
            self.shape = Some(vec![data.len() as u64]);
        }
        self
    }

    pub fn with_i32_data(&mut self, data: &[i32]) -> &mut Self {
        self.datatype = Some(make_i32_type());
        let mut b = Vec::with_capacity(data.len() * 4);
        for &v in data {
            b.extend_from_slice(&v.to_le_bytes());
        }
        self.data = Some(b);
        if self.shape.is_none() {
            self.shape = Some(vec![data.len() as u64]);
        }
        self
    }

    pub fn with_i64_data(&mut self, data: &[i64]) -> &mut Self {
        self.datatype = Some(make_i64_type());
        let mut b = Vec::with_capacity(data.len() * 8);
        for &v in data {
            b.extend_from_slice(&v.to_le_bytes());
        }
        self.data = Some(b);
        if self.shape.is_none() {
            self.shape = Some(vec![data.len() as u64]);
        }
        self
    }

    pub fn with_u8_data(&mut self, data: &[u8]) -> &mut Self {
        self.datatype = Some(make_u8_type());
        self.data = Some(data.to_vec());
        if self.shape.is_none() {
            self.shape = Some(vec![data.len() as u64]);
        }
        self
    }

    pub fn with_i8_data(&mut self, data: &[i8]) -> &mut Self {
        self.datatype = Some(make_i8_type());
        let mut b = Vec::with_capacity(data.len());
        for &v in data {
            b.push(v as u8);
        }
        self.data = Some(b);
        if self.shape.is_none() {
            self.shape = Some(vec![data.len() as u64]);
        }
        self
    }

    pub fn with_i16_data(&mut self, data: &[i16]) -> &mut Self {
        self.datatype = Some(make_i16_type());
        let mut b = Vec::with_capacity(data.len() * 2);
        for &v in data {
            b.extend_from_slice(&v.to_le_bytes());
        }
        self.data = Some(b);
        if self.shape.is_none() {
            self.shape = Some(vec![data.len() as u64]);
        }
        self
    }

    pub fn with_u16_data(&mut self, data: &[u16]) -> &mut Self {
        self.datatype = Some(make_u16_type());
        let mut b = Vec::with_capacity(data.len() * 2);
        for &v in data {
            b.extend_from_slice(&v.to_le_bytes());
        }
        self.data = Some(b);
        if self.shape.is_none() {
            self.shape = Some(vec![data.len() as u64]);
        }
        self
    }

    pub fn with_u32_data(&mut self, data: &[u32]) -> &mut Self {
        self.datatype = Some(make_u32_type());
        let mut b = Vec::with_capacity(data.len() * 4);
        for &v in data {
            b.extend_from_slice(&v.to_le_bytes());
        }
        self.data = Some(b);
        if self.shape.is_none() {
            self.shape = Some(vec![data.len() as u64]);
        }
        self
    }

    pub fn with_u64_data(&mut self, data: &[u64]) -> &mut Self {
        self.datatype = Some(make_u64_type());
        let mut b = Vec::with_capacity(data.len() * 8);
        for &v in data {
            b.extend_from_slice(&v.to_le_bytes());
        }
        self.data = Some(b);
        if self.shape.is_none() {
            self.shape = Some(vec![data.len() as u64]);
        }
        self
    }

    /// Write an object reference dataset. Each address is an 8-byte absolute
    /// address pointing to an object header in the file.
    pub fn with_reference_data(&mut self, addresses: &[u64]) -> &mut Self {
        self.datatype = Some(make_object_reference_type());
        let mut b = Vec::with_capacity(addresses.len() * 8);
        for &addr in addresses {
            b.extend_from_slice(&addr.to_le_bytes());
        }
        self.data = Some(b);
        if self.shape.is_none() {
            self.shape = Some(vec![addresses.len() as u64]);
        }
        self
    }

    /// Write an object reference dataset by path. During file serialization,
    /// each path is resolved to the absolute address of the named object.
    /// Paths use `/` separators (e.g., `"#refs#/child1"`).
    pub fn with_path_references(&mut self, paths: &[&str]) -> &mut Self {
        let targets = paths
            .iter()
            .map(|s| ObjectRefTarget::Path(s.to_string()))
            .collect();
        self.with_object_references(targets)
    }

    /// Write an object-reference dataset from explicit per-element targets,
    /// preserving null/undefined references verbatim. The datatype is set to the
    /// 8-byte object-reference type; each [`ObjectRefTarget::Path`] is resolved to
    /// its destination address during serialization, while
    /// [`ObjectRefTarget::Raw`] is written as-is. This is the faithful re-emit
    /// path used by repack. The shape defaults to `[targets.len()]` unless
    /// [`with_shape`](Self::with_shape) sets it.
    pub(crate) fn with_object_references(&mut self, targets: Vec<ObjectRefTarget>) -> &mut Self {
        self.datatype = Some(make_object_reference_type());
        // Placeholder zeros — patched once all destination addresses are known.
        self.data = Some(vec![0u8; targets.len() * 8]);
        if self.shape.is_none() {
            self.shape = Some(vec![targets.len() as u64]);
        }
        self.reference_targets = Some(
            targets
                .into_iter()
                .enumerate()
                .map(|(i, target)| ObjectRefPatch {
                    byte_offset: i * 8,
                    target,
                })
                .collect(),
        );
        self
    }

    /// Write a dataset whose datatype *contains* object references without being
    /// one — a compound with a reference member, or an array of them.
    ///
    /// `raw` is the source's element bytes; each [`ObjectRefPatch`] names an
    /// address within them to resolve during serialization. Every other byte is
    /// carried through untouched, so the fixed-size members keep their exact
    /// stored bytes. The shape defaults to `[num_elements]` unless
    /// [`with_shape`](Self::with_shape) sets it.
    pub(crate) fn with_embedded_object_references(
        &mut self,
        datatype: Datatype,
        raw: Vec<u8>,
        num_elements: u64,
        patches: Vec<ObjectRefPatch>,
    ) -> &mut Self {
        self.datatype = Some(datatype);
        self.data = Some(raw);
        if self.shape.is_none() {
            self.shape = Some(vec![num_elements]);
        }
        self.reference_targets = Some(patches);
        self
    }

    /// Write a complex32 (f32 real/imag pair) dataset.
    pub fn with_complex32_data(&mut self, data: &[(f32, f32)]) -> &mut Self {
        self.with_complex_data(data)
    }

    /// Write a complex64 (f64 real/imag pair) dataset.
    pub fn with_complex64_data(&mut self, data: &[(f64, f64)]) -> &mut Self {
        self.with_complex_data(data)
    }

    /// Write a complex dataset of any component width: a two-field `{real,
    /// imag}` compound with `real` at offset 0 and `imag` at
    /// `size_of::<T>()`, which is the layout MATLAB v7.3 uses for a complex
    /// array of the component's class.
    pub(crate) fn with_complex_data<T: ComplexComponent>(&mut self, data: &[(T, T)]) -> &mut Self {
        let ct = CompoundTypeBuilder::new()
            .field("real", T::datatype())
            .field("imag", T::datatype())
            .build();
        let mut raw = Vec::with_capacity(data.len() * 2 * size_of::<T>());
        for &(re, im) in data {
            re.encode_le(&mut raw);
            im.encode_le(&mut raw);
        }
        self.with_compound_data(ct, raw, data.len() as u64)
    }

    /// Write a compound (struct) dataset.
    pub fn with_compound_data(
        &mut self,
        datatype: Datatype,
        raw_data: Vec<u8>,
        num_elements: u64,
    ) -> &mut Self {
        self.with_raw_data(datatype, raw_data, num_elements)
    }

    /// Write a dataset from an explicit datatype and its raw element bytes.
    ///
    /// The lowest-level data entry point: `raw_data` is written verbatim as the
    /// dataset's storage, interpreted by `datatype`, so the caller is
    /// responsible for the bytes matching the datatype's on-disk layout (little
    /// endian, `num_elements` elements each of the datatype's size). It underpins
    /// the typed helpers and lets a captured `(datatype, bytes)` pair — for
    /// example from reading an existing dataset — be re-emitted without a typed
    /// helper. The shape defaults to `[num_elements]` unless
    /// [`with_shape`](Self::with_shape) sets it.
    pub fn with_raw_data(
        &mut self,
        datatype: Datatype,
        raw_data: Vec<u8>,
        num_elements: u64,
    ) -> &mut Self {
        self.datatype = Some(datatype);
        self.data = Some(raw_data);
        if self.shape.is_none() {
            self.shape = Some(vec![num_elements]);
        }
        self
    }

    /// Stage a chunked dataset whose chunks are streamed verbatim from a source
    /// file one at a time, without decoding or re-encoding any chunk and without
    /// holding more than one chunk's bytes in memory.
    ///
    /// Repack's out-of-core verbatim path: `meta` is the per-chunk sizes + filter
    /// masks in dense row-major chunk-grid order (one per slot), and `provider`
    /// yields each chunk's already-compressed bytes on demand at write time. The
    /// destination layout is computed from `meta` alone, so the chunks are never
    /// all resident at once. `pipeline_message` is the source's `FilterPipeline`
    /// message bytes, reused as-is so every filter — including ones this crate
    /// cannot itself apply (ZFP, SZIP, unknown) — is reproduced byte-for-byte.
    /// `dims`/`maxshape`/`chunk_dims`/`element_size` describe the geometry. The
    /// shape defaults to `dims` and the chunk dimensions to `chunk_dims`. The
    /// provider owns its source (e.g. an `Arc<File>`), so this carries no
    /// borrowed lifetime.
    #[allow(clippy::too_many_arguments)]
    pub(crate) fn with_raw_chunks_lazy(
        &mut self,
        datatype: Datatype,
        dims: &[u64],
        maxshape: Option<&[u64]>,
        chunk_dims: &[u64],
        element_size: usize,
        pipeline_message: Option<Vec<u8>>,
        meta: Vec<ChunkMeta>,
        provider: Box<dyn ChunkProvider>,
    ) -> &mut Self {
        // Full uncompressed bytes of one chunk (drives the fixed/extensible-array
        // chunk-size encoding width). Saturating arithmetic matches the writer's
        // overflow discipline; the values come from an already-validated source
        // file, so this only guards against an absurd input rather than a real case.
        let raw_size: u64 = chunk_dims
            .iter()
            .copied()
            .product::<u64>()
            .saturating_mul(element_size as u64);
        self.datatype = Some(datatype);
        if self.shape.is_none() {
            self.shape = Some(dims.to_vec());
        }
        if let Some(ms) = maxshape {
            self.maxshape = Some(ms.to_vec());
        }
        self.chunk_options.chunk_dims = Some(chunk_dims.to_vec());
        self.raw_chunks = Some(RawChunkPayload {
            chunk_dims: chunk_dims.to_vec(),
            element_size,
            raw_size,
            pipeline_message,
            meta,
            provider: core::panic::AssertUnwindSafe(provider),
        });
        self
    }

    /// Safely encode a slice of compound values field by field.
    ///
    /// Built-in implementations support numeric tuples with one through twelve
    /// fields. No Rust struct or tuple padding is copied into the file.
    pub fn with_compound_values<T: CompoundType>(
        &mut self,
        values: &[T],
    ) -> Result<&mut Self, crate::error::FormatError> {
        let datatype = T::datatype()?;
        if !matches!(datatype, Datatype::Compound { .. }) {
            return Err(crate::error::FormatError::TypeMismatch {
                expected: "Compound",
                actual: "non-Compound",
            });
        }
        let element_size = datatype.type_size().to_usize()?;
        if element_size == 0 {
            return Err(crate::error::FormatError::InvalidCompoundSize);
        }
        let mut raw = Vec::with_capacity(values.len().saturating_mul(element_size));
        for value in values {
            let start = raw.len();
            value.encode(&mut raw);
            let actual = raw.len() - start;
            if actual != element_size {
                return Err(crate::error::FormatError::DataSizeMismatch {
                    expected: element_size,
                    actual,
                });
            }
        }
        Ok(self.with_compound_data(datatype, raw, values.len() as u64))
    }

    /// Write an enum dataset with i32 values.
    pub fn with_enum_i32_data(&mut self, datatype: Datatype, values: &[i32]) -> &mut Self {
        self.datatype = Some(datatype);
        let mut raw = Vec::with_capacity(values.len() * 4);
        for &v in values {
            raw.extend_from_slice(&v.to_le_bytes());
        }
        self.data = Some(raw);
        if self.shape.is_none() {
            self.shape = Some(vec![values.len() as u64]);
        }
        self
    }

    /// Write an enum dataset with u8 values.
    pub fn with_enum_u8_data(&mut self, datatype: Datatype, values: &[u8]) -> &mut Self {
        self.datatype = Some(datatype);
        self.data = Some(values.to_vec());
        if self.shape.is_none() {
            self.shape = Some(vec![values.len() as u64]);
        }
        self
    }

    /// Write a variable-length UTF-8 string dataset.
    ///
    /// Each element is stored as a global-heap object holding the string's
    /// bytes; the datatype is `H5T_VLEN { H5T_STRING { STRSIZE=VAR, ASCII or
    /// UTF-8 } }`. The shape defaults to `[values.len()]` unless
    /// [`with_shape`](Self::with_shape) sets it (use that for ND VL strings,
    /// passing `values` in row-major order).
    ///
    /// Empty strings become zero-length heap objects (reading back as `""`).
    /// This convenience method cannot distinguish a null element from an empty
    /// one, nor carry embedded NULs, non-UTF-8 payloads, or a specific
    /// charset/padding.
    pub fn with_vlen_strings(&mut self, values: &[&str]) -> &mut Self {
        let datatype = make_vlen_string_type(CharacterSet::Utf8);
        let elements: Vec<VlStringElement> = values
            .iter()
            .map(|s| VlStringElement::Bytes(s.as_bytes().to_vec()))
            .collect();
        self.stage_vlen_strings(datatype, &elements);
        self
    }

    /// Write a variable-length string dataset from an explicit source datatype
    /// and per-element byte payloads, preserving the null-vs-empty distinction.
    ///
    /// `datatype` must be a string-shaped variable-length datatype
    /// (`is_string: true`, or the MATLAB `H5T_VLEN { H5T_STRING { STRSIZE=1 } }`
    /// shape); its charset, padding, and base type are reproduced verbatim. Each
    /// [`VlStringElement`] is either a null reference or a heap object holding
    /// exact bytes. This is the faithful re-emit path used by repack. Returns a
    /// [`TypeMismatch`](crate::FormatError::TypeMismatch) if `datatype` is not a
    /// VL-string datatype. The shape defaults to `[elements.len()]` unless
    /// [`with_shape`](Self::with_shape) sets it.
    pub(crate) fn with_vlen_string_elements(
        &mut self,
        datatype: Datatype,
        elements: &[VlStringElement],
    ) -> Result<&mut Self, crate::error::FormatError> {
        if !crate::vl_data::is_vlen_string_datatype(&datatype) {
            return Err(crate::error::FormatError::TypeMismatch {
                expected: "VariableLength string",
                actual: "non-VariableLength string",
            });
        }
        self.stage_vlen_strings(datatype, elements);
        Ok(self)
    }

    /// Shared body of the VL-string write entry points: stage the references
    /// and global heap collection and record them on the builder.
    fn stage_vlen_strings(&mut self, datatype: Datatype, elements: &[VlStringElement]) {
        // A VL string's base type is one byte, so the reference length is the
        // byte count (element_size = 1).
        self.stage_vlen_elements(datatype, elements, 1);
    }

    /// Write a *non-string* variable-length (sequence) dataset from an explicit
    /// source datatype and per-element byte payloads.
    ///
    /// `datatype` must be a non-string VL datatype (e.g. `H5T_VLEN
    /// { H5T_NATIVE_DOUBLE }`); its base type is reproduced verbatim. Each
    /// element's exact heap bytes are re-staged through a fresh global heap, and
    /// the per-element reference stores the base-type element count. This is the
    /// faithful re-emit path used by repack. Returns a
    /// [`TypeMismatch`](crate::FormatError::TypeMismatch) if `datatype` is a
    /// string-shaped VL datatype or not variable-length at all. The shape
    /// defaults to `[elements.len()]` unless [`with_shape`](Self::with_shape)
    /// sets it.
    pub(crate) fn with_vlen_sequence_elements(
        &mut self,
        datatype: Datatype,
        elements: &[VlStringElement],
    ) -> Result<&mut Self, crate::error::FormatError> {
        let Datatype::VariableLength { base_type, .. } = &datatype else {
            return Err(crate::error::FormatError::TypeMismatch {
                expected: "non-string VariableLength",
                actual: "non-VariableLength",
            });
        };
        if crate::vl_data::is_vlen_string_datatype(&datatype) {
            return Err(crate::error::FormatError::TypeMismatch {
                expected: "non-string VariableLength",
                actual: "VariableLength string",
            });
        }
        let element_size = base_type.type_size() as usize;
        if element_size == 0 {
            return Err(crate::error::FormatError::VlDataError(
                "non-string VL base type has zero size".into(),
            ));
        }
        self.stage_vlen_elements(datatype, elements, element_size);
        Ok(self)
    }

    /// Write a dataset whose datatype *contains* variable-length members without
    /// being variable-length itself — a compound with a VL member, or an array of
    /// such compounds.
    ///
    /// `raw` is the source's element bytes and `offsets` gives the byte offset of
    /// every embedded VL reference within them, paired in order with `elements`
    /// (each either a null reference or the exact heap bytes it named). The
    /// references are re-stamped to point at this file's own global heap, so the
    /// source's addresses never survive into the output. This is the faithful
    /// re-emit path used by repack; the shape defaults to `[num_elements]` unless
    /// [`with_shape`](Self::with_shape) sets it.
    pub(crate) fn with_embedded_vlen_elements(
        &mut self,
        datatype: Datatype,
        raw: Vec<u8>,
        num_elements: u64,
        offsets: &[usize],
        elements: &[VlStringElement],
    ) -> &mut Self {
        let staging = stage_embedded_vl_elements(raw, offsets, elements);
        self.datatype = Some(datatype);
        self.data = Some(staging.refs.clone());
        self.vl_string_staging = Some(staging);
        if self.shape.is_none() {
            self.shape = Some(vec![num_elements]);
        }
        self
    }

    /// Shared body of the VL write entry points (string and sequence): stage the
    /// references and global heap collection and record them on the builder.
    fn stage_vlen_elements(
        &mut self,
        datatype: Datatype,
        elements: &[VlStringElement],
        element_size: usize,
    ) {
        let n = elements.len() as u64;
        let staging = stage_vl_elements(elements, element_size);
        self.datatype = Some(datatype);
        self.data = Some(staging.refs.clone());
        self.vl_string_staging = Some(staging);
        if self.shape.is_none() {
            self.shape = Some(vec![n]);
        }
    }

    /// Write an array-typed dataset.
    pub fn with_array_data(
        &mut self,
        base_type: Datatype,
        array_dims: &[u32],
        raw_data: Vec<u8>,
        num_elements: u64,
    ) -> &mut Self {
        self.datatype = Some(Datatype::Array {
            base_type: Box::new(base_type),
            dimensions: array_dims.to_vec(),
        });
        self.data = Some(raw_data);
        if self.shape.is_none() {
            self.shape = Some(vec![num_elements]);
        }
        self
    }

    pub fn with_shape(&mut self, shape: &[u64]) -> &mut Self {
        self.shape = Some(shape.to_vec());
        self
    }

    /// Set the datatype without providing data.
    /// Use with `with_shape` for empty/zero-dimension datasets.
    pub fn with_dtype(&mut self, dt: Datatype) -> &mut Self {
        self.datatype = Some(dt);
        self
    }

    /// Set maximum dimensions for a resizable dataset.
    /// Use `u64::MAX` for unlimited dimensions.
    pub fn with_maxshape(&mut self, maxshape: &[u64]) -> &mut Self {
        self.maxshape = Some(maxshape.to_vec());
        self
    }

    pub fn set_attr(&mut self, name: &str, value: AttrValue) -> &mut Self {
        self.attrs.push((name.to_string(), value));
        self
    }

    /// Enable chunked storage with given chunk dimensions.
    pub fn with_chunks(&mut self, chunk_dims: &[u64]) -> &mut Self {
        self.chunk_options.chunk_dims = Some(chunk_dims.to_vec());
        self
    }

    /// Enable deflate compression (implies chunked if not already set).
    pub fn with_deflate(&mut self, level: u32) -> &mut Self {
        self.chunk_options.deflate_level = Some(level);
        self
    }

    /// Enable shuffle filter (usually combined with deflate).
    pub fn with_shuffle(&mut self) -> &mut Self {
        self.chunk_options.shuffle = true;
        self
    }

    /// Enable fletcher32 checksum.
    pub fn with_fletcher32(&mut self) -> &mut Self {
        self.chunk_options.fletcher32 = true;
        self
    }

    /// Enable scale-offset compression (implies chunked if not already set).
    ///
    /// Scale-offset stores each chunk's values as offsets from the chunk
    /// minimum, packed into the fewest bits the chunk's range needs:
    ///
    /// * [`ScaleOffset::Integer`] is **lossless** for integer datasets. Pass
    ///   `0` to let the encoder choose the bit width per chunk (the usual
    ///   choice).
    /// * [`ScaleOffset::FloatDScale`] is **lossy** for float datasets: values
    ///   are rounded to the given number of decimal digits before packing.
    ///
    /// The datatype class/sign/byte-order are derived from the dataset's
    /// datatype when the file is written, so the mode must match the data
    /// (integer mode on `with_i*`/`with_u*` data, float mode on
    /// `with_f32`/`with_f64` data) or `finish()` / `write()` returns a
    /// [`FormatError`](crate::FormatError). Scale-offset is mutually exclusive
    /// with ZFP and replaces shuffle, but may be combined with
    /// [`with_deflate`](Self::with_deflate). Files are readable by the
    /// reference HDF5 library (filter id 6) and vice versa.
    pub fn with_scale_offset(&mut self, mode: ScaleOffset) -> &mut Self {
        self.chunk_options.scale_offset = Some(mode);
        self
    }

    /// Enable ZFP fixed-rate compression (implies chunked if not already set).
    ///
    /// `rate` is the number of compressed bits per value. Supports f32, f64,
    /// i32, and i64 datasets in 1D–4D. When ZFP is active it replaces shuffle
    /// and deflate on the same dataset.
    ///
    /// The scalar type is derived from the dataset's datatype when the file
    /// is written, so any of `with_{f32,f64,i32,i64}_data` or an explicit
    /// `with_dtype` establishes it. `finish()` / `write()` returns
    /// [`FormatError::UnsupportedZfp`](crate::FormatError::UnsupportedZfp) if
    /// the dataset's datatype isn't one of the four supported scalar types,
    /// or if the chunk rank is outside 1..=4.
    ///
    /// The resulting file is byte-compatible with the reference H5Z-ZFP
    /// plugin (HDF5 filter ID 32013): other tools like h5py + hdf5plugin
    /// will read and decompress it, and vice versa.
    #[cfg(feature = "zfp")]
    pub fn with_zfp(&mut self, rate: f64) -> &mut Self {
        self.chunk_options.zfp_rate = Some(rate);
        self
    }

    /// Attach SHINES provenance metadata (SHA-256, creator, timestamp).
    ///
    /// The SHA-256 hash of the raw dataset bytes is computed automatically
    /// during file serialization and stored as `_provenance_sha256`.
    #[cfg(feature = "provenance")]
    pub fn with_provenance(
        &mut self,
        creator: &str,
        timestamp: &str,
        source: Option<&str>,
    ) -> &mut Self {
        self.provenance = Some(ProvenanceConfig {
            creator: creator.to_string(),
            timestamp: timestamp.to_string(),
            source: source.map(|s| s.to_string()),
        });
        self
    }
}

// ---- Group builder ----

/// Builder for HDF5 groups.
///
/// Datasets, sub-groups, and attributes can be added in any order before
/// calling [`finish()`](GroupBuilder::finish). This is useful when the full
/// set of attributes is not known up front — for example, building a
/// MATLAB struct where `MATLAB_fields` lists every child dataset name:
///
/// ```rust
/// # use hdf5_pure::{FileBuilder, AttrValue};
/// let mut builder = FileBuilder::new();
/// let mut grp = builder.create_group("my_struct");
///
/// let mut fields = Vec::new();
/// for name in &["x", "y", "z"] {
///     fields.push(name.to_string());
///     grp.create_dataset(name).with_f64_data(&[0.0]);
/// }
///
/// // Attribute set after all children are created
/// grp.set_attr("MATLAB_fields", AttrValue::VarLenAsciiArray(fields));
/// builder.add_group(grp.finish());
/// ```
pub struct GroupBuilder {
    pub(crate) name: String,
    pub(crate) datasets: Vec<DatasetBuilder>,
    pub(crate) sub_groups: Vec<FinishedGroup>,
    pub(crate) attrs: Vec<(String, AttrValue)>,
}

impl GroupBuilder {
    pub(crate) fn new(name: &str) -> Self {
        Self {
            name: name.to_string(),
            datasets: Vec::new(),
            sub_groups: Vec::new(),
            attrs: Vec::new(),
        }
    }

    pub fn create_dataset(&mut self, name: &str) -> &mut DatasetBuilder {
        self.datasets.push(DatasetBuilder::new(name));
        self.datasets.last_mut().unwrap()
    }

    /// Create a nested group builder. Call `.finish()` on it and then
    /// `add_group()` to add it to this group.
    pub fn create_group(&mut self, name: &str) -> GroupBuilder {
        GroupBuilder::new(name)
    }

    /// Add a finished sub-group to this group.
    pub fn add_group(&mut self, group: FinishedGroup) {
        self.sub_groups.push(group);
    }

    pub fn set_attr(&mut self, name: &str, value: AttrValue) {
        self.attrs.push((name.to_string(), value));
    }

    /// Consume the builder, returning a FinishedGroup to add to FileWriter.
    pub fn finish(self) -> FinishedGroup {
        FinishedGroup {
            name: self.name,
            datasets: self.datasets,
            sub_groups: self.sub_groups,
            attrs: self.attrs,
        }
    }
}

/// A finished group ready for the file writer.
pub struct FinishedGroup {
    pub(crate) name: String,
    pub(crate) datasets: Vec<DatasetBuilder>,
    pub(crate) sub_groups: Vec<FinishedGroup>,
    pub(crate) attrs: Vec<(String, AttrValue)>,
}