rustyhdf5_format/
metadata_index.rs

1//! Index-table + metadata-block architecture for independent parallel dataset creation.
2//!
3//! Based on the approach from "Parallel Data Object Creation: Scalable Metadata Management"
4//! (arxiv 2506.15114). Each creator independently builds a [`MetadataBlock`] containing
5//! dataset metadata. Blocks are merged into a [`MetadataIndex`] without collective
6//! synchronization, enabling concurrent dataset creation.
7
8#[cfg(not(feature = "std"))]
9use alloc::{string::String, string::ToString, vec::Vec};
10
11#[cfg(not(feature = "std"))]
12extern crate alloc;
13
14use crate::chunked_write::ChunkOptions;
15use crate::dataspace::{Dataspace, DataspaceType};
16use crate::datatype::Datatype;
17use crate::error::FormatError;
18use crate::type_builders::AttrValue;
19
20/// A single dataset's metadata collected independently by one creator.
21#[derive(Debug, Clone)]
22pub struct DatasetMetadata {
23    /// Dataset name (path component, not full path).
24    pub name: String,
25    /// HDF5 datatype descriptor.
26    pub datatype: Datatype,
27    /// HDF5 dataspace (shape, rank, max dimensions).
28    pub dataspace: Dataspace,
29    /// Chunk layout options (chunk dims, compression, etc.).
30    pub chunk_options: ChunkOptions,
31    /// Optional maximum shape for resizable datasets.
32    pub maxshape: Option<Vec<u64>>,
33    /// User-defined attributes on this dataset.
34    pub attrs: Vec<(String, AttrValue)>,
35    /// Raw data bytes for this dataset.
36    pub raw_data: Vec<u8>,
37}
38
39/// Metadata created independently by a single creator (e.g. one thread).
40///
41/// In independent mode, each creator accumulates datasets into its own block.
42/// Blocks are later merged into a [`MetadataIndex`].
43#[derive(Debug, Clone)]
44pub struct MetadataBlock {
45    /// Unique identifier for the creator (e.g. thread index).
46    pub creator_id: u32,
47    /// Datasets defined by this creator.
48    pub datasets: Vec<DatasetMetadata>,
49}
50
51impl MetadataBlock {
52    /// Create a new empty metadata block for the given creator.
53    pub fn new(creator_id: u32) -> Self {
54        Self {
55            creator_id,
56            datasets: Vec::new(),
57        }
58    }
59
60    /// Add a dataset to this block.
61    pub fn add_dataset(&mut self, meta: DatasetMetadata) {
62        self.datasets.push(meta);
63    }
64}
65
66/// Creation mode for metadata blocks.
67#[derive(Debug, Clone, Copy, PartialEq, Eq)]
68pub enum CreationMode {
69    /// All metadata in one block (traditional collective approach).
70    Collective,
71    /// Per-creator blocks merged at finalization (parallel independent approach).
72    Independent,
73}
74
75/// Index table that maps dataset names to their metadata block locations.
76///
77/// After merging, each entry records which block contributed each dataset
78/// and the dataset's position within the merged sequence.
79#[derive(Debug, Clone)]
80pub struct IndexEntry {
81    /// Dataset name.
82    pub name: String,
83    /// Index into the flattened dataset list.
84    pub dataset_index: usize,
85    /// Which creator block this came from.
86    pub source_block: u32,
87}
88
89/// The merged metadata index produced from one or more [`MetadataBlock`]s.
90#[derive(Debug, Clone)]
91pub struct MetadataIndex {
92    /// Creation mode used.
93    pub mode: CreationMode,
94    /// Ordered index entries (sorted by name for deterministic output).
95    pub entries: Vec<IndexEntry>,
96    /// Flattened list of all dataset metadata, in entry order.
97    pub datasets: Vec<DatasetMetadata>,
98}
99
100impl MetadataIndex {
101    /// Create a collective-mode index from a single block of datasets.
102    pub fn from_collective(datasets: Vec<DatasetMetadata>) -> Result<Self, FormatError> {
103        let mut entries = Vec::with_capacity(datasets.len());
104        for (i, ds) in datasets.iter().enumerate() {
105            entries.push(IndexEntry {
106                name: ds.name.clone(),
107                dataset_index: i,
108                source_block: 0,
109            });
110        }
111        // Check for duplicates
112        check_duplicates(&entries)?;
113        Ok(Self {
114            mode: CreationMode::Collective,
115            entries,
116            datasets,
117        })
118    }
119
120    /// Merge multiple independently created metadata blocks into a single index.
121    ///
122    /// Returns an error if any two blocks contain datasets with the same name.
123    pub fn merge_blocks(blocks: &[MetadataBlock]) -> Result<Self, FormatError> {
124        let total: usize = blocks.iter().map(|b| b.datasets.len()).sum();
125        let mut entries = Vec::with_capacity(total);
126        let mut datasets = Vec::with_capacity(total);
127        let mut idx = 0;
128
129        for block in blocks {
130            for ds in &block.datasets {
131                entries.push(IndexEntry {
132                    name: ds.name.clone(),
133                    dataset_index: idx,
134                    source_block: block.creator_id,
135                });
136                datasets.push(ds.clone());
137                idx += 1;
138            }
139        }
140
141        // Sort entries by name for deterministic file layout
142        let mut order: Vec<usize> = (0..entries.len()).collect();
143        order.sort_by(|&a, &b| entries[a].name.cmp(&entries[b].name));
144
145        let sorted_entries: Vec<IndexEntry> = order
146            .iter()
147            .enumerate()
148            .map(|(new_idx, &old_idx)| IndexEntry {
149                name: entries[old_idx].name.clone(),
150                dataset_index: new_idx,
151                source_block: entries[old_idx].source_block,
152            })
153            .collect();
154        let sorted_datasets: Vec<DatasetMetadata> =
155            order.iter().map(|&i| datasets[i].clone()).collect();
156
157        check_duplicates(&sorted_entries)?;
158
159        Ok(Self {
160            mode: CreationMode::Independent,
161            entries: sorted_entries,
162            datasets: sorted_datasets,
163        })
164    }
165
166    /// Look up a dataset by name.
167    pub fn find(&self, name: &str) -> Option<&DatasetMetadata> {
168        self.entries
169            .iter()
170            .find(|e| e.name == name)
171            .map(|e| &self.datasets[e.dataset_index])
172    }
173
174    /// Number of datasets in the index.
175    pub fn len(&self) -> usize {
176        self.datasets.len()
177    }
178
179    /// Whether the index is empty.
180    pub fn is_empty(&self) -> bool {
181        self.datasets.is_empty()
182    }
183}
184
185/// Helper: build a `DatasetMetadata` from common parameters.
186pub fn build_dataset_metadata(
187    name: &str,
188    datatype: Datatype,
189    shape: Vec<u64>,
190    raw_data: Vec<u8>,
191    chunk_options: ChunkOptions,
192    maxshape: Option<Vec<u64>>,
193    attrs: Vec<(String, AttrValue)>,
194) -> DatasetMetadata {
195    let dataspace = Dataspace {
196        space_type: if shape.is_empty() {
197            DataspaceType::Scalar
198        } else {
199            DataspaceType::Simple
200        },
201        rank: shape.len() as u8,
202        dimensions: shape,
203        max_dimensions: maxshape.clone(),
204    };
205    DatasetMetadata {
206        name: name.to_string(),
207        datatype,
208        dataspace,
209        chunk_options,
210        maxshape,
211        attrs,
212        raw_data,
213    }
214}
215
216fn check_duplicates(entries: &[IndexEntry]) -> Result<(), FormatError> {
217    for i in 1..entries.len() {
218        if entries[i].name == entries[i - 1].name {
219            return Err(FormatError::DuplicateDatasetName(entries[i].name.clone()));
220        }
221    }
222    Ok(())
223}
224
225#[cfg(test)]
226mod tests {
227    use super::*;
228    use crate::type_builders::make_f64_type;
229
230    fn sample_meta(name: &str) -> DatasetMetadata {
231        build_dataset_metadata(
232            name,
233            make_f64_type(),
234            vec![3],
235            vec![0u8; 24],
236            ChunkOptions::default(),
237            None,
238            vec![],
239        )
240    }
241
242    #[test]
243    fn collective_mode_basic() {
244        let ds = vec![sample_meta("a"), sample_meta("b")];
245        let idx = MetadataIndex::from_collective(ds).unwrap();
246        assert_eq!(idx.len(), 2);
247        assert_eq!(idx.mode, CreationMode::Collective);
248        assert!(idx.find("a").is_some());
249        assert!(idx.find("b").is_some());
250        assert!(idx.find("c").is_none());
251    }
252
253    #[test]
254    fn merge_two_blocks() {
255        let mut b0 = MetadataBlock::new(0);
256        b0.add_dataset(sample_meta("ds_a"));
257        b0.add_dataset(sample_meta("ds_c"));
258
259        let mut b1 = MetadataBlock::new(1);
260        b1.add_dataset(sample_meta("ds_b"));
261
262        let idx = MetadataIndex::merge_blocks(&[b0, b1]).unwrap();
263        assert_eq!(idx.len(), 3);
264        assert_eq!(idx.entries[0].name, "ds_a");
265        assert_eq!(idx.entries[1].name, "ds_b");
266        assert_eq!(idx.entries[2].name, "ds_c");
267    }
268
269    #[test]
270    fn merge_detects_duplicates() {
271        let mut b0 = MetadataBlock::new(0);
272        b0.add_dataset(sample_meta("shared_name"));
273
274        let mut b1 = MetadataBlock::new(1);
275        b1.add_dataset(sample_meta("shared_name"));
276
277        let err = MetadataIndex::merge_blocks(&[b0, b1]).unwrap_err();
278        assert!(matches!(err, FormatError::DuplicateDatasetName(ref n) if n == "shared_name"));
279    }
280
281    #[test]
282    fn empty_merge() {
283        let idx = MetadataIndex::merge_blocks(&[]).unwrap();
284        assert!(idx.is_empty());
285    }
286
287    #[test]
288    fn single_block_merge() {
289        let mut b = MetadataBlock::new(0);
290        b.add_dataset(sample_meta("only"));
291        let idx = MetadataIndex::merge_blocks(&[b]).unwrap();
292        assert_eq!(idx.len(), 1);
293        assert_eq!(idx.find("only").unwrap().name, "only");
294    }
295}
rustyhdf5_format/metadata_index.rs

rustyhdf5_format/
metadata_index.rs
