hdf5_reader/
group.rs

1use std::sync::Arc;
2
3use crate::attribute_api::{
4    collect_attribute_messages_storage, decoded_vlen_strings_storage, resolve_vlen_bytes_storage,
5    Attribute,
6};
7use crate::btree_v1;
8use crate::btree_v2;
9use crate::checksum::jenkins_lookup3;
10use crate::dataset::Dataset;
11use crate::error::{Error, Result};
12use crate::fractal_heap::{FractalHeap, FractalHeapDirectBlockCache};
13use crate::io::Cursor;
14use crate::local_heap::LocalHeap;
15use crate::messages::datatype::VarLenKind;
16use crate::messages::link::{self, LinkMessage, LinkTarget};
17use crate::messages::link_info::LinkInfoMessage;
18use crate::messages::symbol_table_msg::SymbolTableMessage;
19use crate::messages::HdfMessage;
20use crate::storage::Storage;
21use crate::FileContext;
22
23/// A group within an HDF5 file.
24#[derive(Clone)]
25pub struct Group {
26    context: Arc<FileContext>,
27    pub(crate) name: String,
28    pub(crate) address: u64,
29    /// Address of the root group's object header, used for resolving soft links.
30    pub(crate) root_address: u64,
31}
32
33#[derive(Clone)]
34struct ChildEntry {
35    name: String,
36    location: ObjectLocation,
37}
38
39#[derive(Clone)]
40struct ObjectLocation {
41    context: Arc<FileContext>,
42    address: u64,
43    root_address: u64,
44}
45
46#[derive(Debug, Clone, Copy, PartialEq, Eq)]
47enum ChildObjectKind {
48    Group,
49    Dataset,
50    Other,
51}
52
53impl Group {
54    /// Create a group from a known object header address.
55    pub(crate) fn new(
56        context: Arc<FileContext>,
57        address: u64,
58        name: String,
59        root_address: u64,
60    ) -> Self {
61        Group {
62            context,
63            name,
64            address,
65            root_address,
66        }
67    }
68
69    /// Group name.
70    pub fn name(&self) -> &str {
71        &self.name
72    }
73
74    /// Object header address of this group within the file.
75    pub fn address(&self) -> u64 {
76        self.address
77    }
78
79    /// Materialize the full file backing this group.
80    pub fn file_data(&self) -> Result<crate::storage::StorageBuffer> {
81        self.context.full_file_data()
82    }
83
84    /// Access the underlying random-access storage backend.
85    pub fn storage(&self) -> &dyn Storage {
86        self.context.storage.as_ref()
87    }
88
89    /// Size of file offsets in bytes.
90    pub fn offset_size(&self) -> u8 {
91        self.context.superblock.offset_size
92    }
93
94    /// Size of file lengths in bytes.
95    pub fn length_size(&self) -> u8 {
96        self.context.superblock.length_size
97    }
98
99    /// Parse (or retrieve from cache) the object header at the given address.
100    fn cached_header(&self, addr: u64) -> Result<Arc<crate::object_header::ObjectHeader>> {
101        self.context.get_or_parse_header(addr)
102    }
103
104    fn local_location(&self, address: u64) -> ObjectLocation {
105        ObjectLocation {
106            context: self.context.clone(),
107            address,
108            root_address: self.root_address,
109        }
110    }
111
112    /// List all child groups.
113    pub fn groups(&self) -> Result<Vec<Group>> {
114        let (groups, _) = self.resolve_member_objects()?;
115        Ok(groups)
116    }
117
118    /// List all child members, partitioned into groups and datasets.
119    pub fn members(&self) -> Result<(Vec<Group>, Vec<Dataset>)> {
120        self.resolve_member_objects()
121    }
122
123    fn resolve_member_objects(&self) -> Result<(Vec<Group>, Vec<Dataset>)> {
124        let children = self.resolve_children()?;
125        let mut groups = Vec::new();
126        let mut datasets = Vec::new();
127        for child in &children {
128            match self.child_object_kind(child)? {
129                ChildObjectKind::Group => {
130                    groups.push(Group::new(
131                        child.location.context.clone(),
132                        child.location.address,
133                        child.name.clone(),
134                        child.location.root_address,
135                    ));
136                }
137                ChildObjectKind::Dataset => {
138                    if let Some(dataset) = self.try_open_child_dataset(child)? {
139                        datasets.push(dataset);
140                    }
141                }
142                ChildObjectKind::Other => {}
143            }
144        }
145        Ok((groups, datasets))
146    }
147
148    /// Get a child group by name.
149    pub fn group(&self, name: &str) -> Result<Group> {
150        if let Some(child) = self.resolve_child(name)? {
151            return match self.child_object_kind(&child)? {
152                ChildObjectKind::Group => Ok(Group::new(
153                    child.location.context.clone(),
154                    child.location.address,
155                    child.name.clone(),
156                    child.location.root_address,
157                )),
158                ChildObjectKind::Dataset => Err(Error::GroupNotFound(format!(
159                    "'{}' is a dataset, not a group",
160                    name
161                ))),
162                ChildObjectKind::Other => {
163                    Err(Error::GroupNotFound(format!("'{}' is not a group", name)))
164                }
165            };
166        }
167        Err(Error::GroupNotFound(name.to_string()))
168    }
169
170    /// List all child datasets.
171    pub fn datasets(&self) -> Result<Vec<Dataset>> {
172        let (_, datasets) = self.resolve_member_objects()?;
173        Ok(datasets)
174    }
175
176    /// Get a child dataset by name.
177    pub fn dataset(&self, name: &str) -> Result<Dataset> {
178        if let Some(child) = self.resolve_child(name)? {
179            if let Some(dataset) = self.try_open_child_dataset(&child)? {
180                return Ok(dataset);
181            }
182            return Err(Error::DatasetNotFound(name.to_string()));
183        }
184        Err(Error::DatasetNotFound(name.to_string()))
185    }
186
187    /// List attributes on this group.
188    pub fn attributes(&self) -> Result<Vec<Attribute>> {
189        let mut header = (*self.cached_header(self.address)?).clone();
190        header.resolve_shared_messages_storage(
191            self.context.storage.as_ref(),
192            self.offset_size(),
193            self.length_size(),
194        )?;
195        Ok(collect_attribute_messages_storage(
196            &header,
197            self.context.storage.as_ref(),
198            self.offset_size(),
199            self.length_size(),
200            Some(self.context.filter_registry.as_ref()),
201        )?
202        .into_iter()
203        .map(|attr| {
204            let element_count = attr.dataspace.num_elements().ok();
205            let decoded_strings = element_count.and_then(|element_count| {
206                decoded_vlen_strings_storage(
207                    &attr.datatype,
208                    &attr.raw_data,
209                    self.context.storage.as_ref(),
210                    self.offset_size(),
211                    self.length_size(),
212                    element_count,
213                )
214            });
215            let raw_data = match &attr.datatype {
216                crate::messages::datatype::Datatype::VarLen {
217                    base,
218                    kind: VarLenKind::String,
219                    ..
220                } if matches!(
221                    base.as_ref(),
222                    crate::messages::datatype::Datatype::FixedPoint { size: 1, .. }
223                ) && matches!(attr.dataspace.num_elements(), Ok(1)) =>
224                {
225                    resolve_vlen_bytes_storage(
226                        &attr.raw_data,
227                        self.context.storage.as_ref(),
228                        self.offset_size(),
229                        self.length_size(),
230                    )
231                    .unwrap_or_else(|| attr.raw_data.clone())
232                }
233                _ => attr.raw_data.clone(),
234            };
235            Attribute {
236                name: attr.name,
237                datatype: attr.datatype,
238                shape: match attr.dataspace.dataspace_type {
239                    crate::messages::dataspace::DataspaceType::Scalar => vec![],
240                    crate::messages::dataspace::DataspaceType::Null => vec![0],
241                    crate::messages::dataspace::DataspaceType::Simple => attr.dataspace.dims,
242                },
243                raw_data,
244                decoded_strings,
245            }
246        })
247        .collect())
248    }
249
250    /// Find an attribute by name.
251    pub fn attribute(&self, name: &str) -> Result<Attribute> {
252        let attrs = self.attributes()?;
253        attrs
254            .into_iter()
255            .find(|a| a.name == name)
256            .ok_or_else(|| Error::AttributeNotFound(name.to_string()))
257    }
258
259    /// Resolve children from the object header.
260    /// Handles both old-style (symbol table) and new-style (link messages) groups.
261    fn resolve_children(&self) -> Result<Vec<ChildEntry>> {
262        self.resolve_children_with_link_depth(0)
263    }
264
265    fn resolve_child(&self, name: &str) -> Result<Option<ChildEntry>> {
266        self.resolve_child_with_link_depth(name, 0)
267    }
268
269    fn resolve_child_with_link_depth(
270        &self,
271        name: &str,
272        link_depth: u32,
273    ) -> Result<Option<ChildEntry>> {
274        let header = self.cached_header(self.address)?;
275
276        let mut link_info: Option<LinkInfoMessage> = None;
277        let mut matching_compact_link: Option<LinkMessage> = None;
278
279        for msg in &header.messages {
280            match msg {
281                HdfMessage::SymbolTable(st) => {
282                    return Ok(self
283                        .resolve_old_style_group_storage(st)?
284                        .into_iter()
285                        .find(|child| child.name == name));
286                }
287                HdfMessage::Link(link) if link.name == name => {
288                    matching_compact_link = Some(link.clone());
289                }
290                HdfMessage::LinkInfo(li) => {
291                    link_info = Some(li.clone());
292                }
293                _ => {}
294            }
295        }
296
297        if let Some(link) = matching_compact_link {
298            if let Some(child) = self.resolve_link_message_target(&link, link_depth)? {
299                return Ok(Some(child));
300            }
301        }
302
303        if let Some(ref li) = link_info {
304            if !Cursor::is_undefined_offset(li.fractal_heap_address, self.offset_size()) {
305                return self.resolve_dense_link_storage(li, name, link_depth);
306            }
307        }
308
309        Ok(None)
310    }
311
312    /// Resolve children with a soft-link depth counter to prevent cycles.
313    fn resolve_children_with_link_depth(&self, link_depth: u32) -> Result<Vec<ChildEntry>> {
314        let header = self.cached_header(self.address)?;
315
316        let mut children = Vec::new();
317
318        // Check for old-style groups (symbol table message)
319        let mut found_symbol_table = false;
320        // Check for new-style groups (link messages)
321        let mut link_info: Option<LinkInfoMessage> = None;
322        let mut links: Vec<LinkMessage> = Vec::new();
323
324        for msg in &header.messages {
325            match msg {
326                HdfMessage::SymbolTable(st) => {
327                    found_symbol_table = true;
328                    children = self.resolve_old_style_group_storage(st)?;
329                }
330                HdfMessage::Link(link) => {
331                    links.push(link.clone());
332                }
333                HdfMessage::LinkInfo(li) => {
334                    link_info = Some(li.clone());
335                }
336                _ => {}
337            }
338        }
339
340        if !found_symbol_table {
341            // New-style group: use compact links from header messages
342            self.resolve_link_targets(&links, link_depth, &mut children)?;
343
344            // Dense-link storage can coexist with compact links, so merge both.
345            if let Some(ref li) = link_info {
346                if !Cursor::is_undefined_offset(li.fractal_heap_address, self.offset_size()) {
347                    for child in self.resolve_dense_links_storage(li, link_depth)? {
348                        let is_duplicate = children.iter().any(|existing| {
349                            existing.name == child.name
350                                && existing.location.address == child.location.address
351                                && Arc::ptr_eq(&existing.location.context, &child.location.context)
352                        });
353                        if !is_duplicate {
354                            children.push(child);
355                        }
356                    }
357                }
358            }
359        }
360
361        Ok(children)
362    }
363
364    /// Resolve link targets (hard and soft), appending to `children`.
365    fn resolve_link_targets(
366        &self,
367        links: &[LinkMessage],
368        link_depth: u32,
369        children: &mut Vec<ChildEntry>,
370    ) -> Result<()> {
371        for link in links {
372            if let Some(child) = self.resolve_link_message_target(link, link_depth)? {
373                children.push(child);
374            }
375        }
376        Ok(())
377    }
378
379    fn resolve_link_message_target(
380        &self,
381        link: &LinkMessage,
382        link_depth: u32,
383    ) -> Result<Option<ChildEntry>> {
384        match &link.target {
385            LinkTarget::Hard { address } => Ok(Some(ChildEntry {
386                name: link.name.clone(),
387                location: self.local_location(*address),
388            })),
389            LinkTarget::Soft { path } => Ok(self
390                .resolve_soft_link_depth(path, link_depth)
391                .ok()
392                .map(|location| ChildEntry {
393                    name: link.name.clone(),
394                    location,
395                })),
396            LinkTarget::External { filename, path } => Ok(self
397                .resolve_external_link_depth(filename, path, link_depth)?
398                .map(|location| ChildEntry {
399                    name: link.name.clone(),
400                    location,
401                })),
402        }
403    }
404
405    fn resolve_old_style_group_storage(&self, st: &SymbolTableMessage) -> Result<Vec<ChildEntry>> {
406        let heap = LocalHeap::parse_at_storage(
407            self.context.storage.as_ref(),
408            st.heap_address,
409            self.offset_size(),
410            self.length_size(),
411        )?;
412
413        let leaves = btree_v1::collect_btree_v1_leaves_storage(
414            self.context.storage.as_ref(),
415            st.btree_address,
416            self.offset_size(),
417            self.length_size(),
418            None,
419            &[],
420            None,
421        )?;
422
423        let mut children = Vec::new();
424        for (_key, snod_address) in &leaves {
425            let header_len = 8 + 2 * usize::from(self.offset_size());
426            let prefix = self.context.read_range(*snod_address, header_len)?;
427            let mut prefix_cursor = Cursor::new(prefix.as_ref());
428            let sig = prefix_cursor.read_bytes(4)?;
429            if sig != *b"SNOD" {
430                return Err(Error::InvalidData(format!(
431                    "expected SNOD signature at offset {:#x}",
432                    snod_address
433                )));
434            }
435            let version = prefix_cursor.read_u8()?;
436            if version != 1 {
437                return Err(Error::InvalidData(format!(
438                    "unsupported symbol table node version {}",
439                    version
440                )));
441            }
442            prefix_cursor.skip(1)?;
443            let num_symbols = prefix_cursor.read_u16_le()?;
444            let node_len =
445                8 + usize::from(num_symbols) * (2 * usize::from(self.offset_size()) + 4 + 4 + 16);
446            let bytes = self.context.read_range(*snod_address, node_len)?;
447            let mut cursor = Cursor::new(bytes.as_ref());
448            let snod = crate::symbol_table::SymbolTableNode::parse(
449                &mut cursor,
450                self.offset_size(),
451                self.length_size(),
452            )?;
453
454            for entry in &snod.entries {
455                let name =
456                    heap.get_string_storage(entry.link_name_offset, self.context.storage.as_ref())?;
457                children.push(ChildEntry {
458                    name,
459                    location: self.local_location(entry.object_header_address),
460                });
461            }
462        }
463
464        Ok(children)
465    }
466
467    fn resolve_dense_links_storage(
468        &self,
469        link_info: &LinkInfoMessage,
470        link_depth: u32,
471    ) -> Result<Vec<ChildEntry>> {
472        let heap = FractalHeap::parse_at_storage(
473            self.context.storage.as_ref(),
474            link_info.fractal_heap_address,
475            self.offset_size(),
476            self.length_size(),
477        )?;
478
479        let btree_header = btree_v2::BTreeV2Header::parse_at_storage(
480            self.context.storage.as_ref(),
481            link_info.btree_name_index_address,
482            self.offset_size(),
483            self.length_size(),
484        )?;
485
486        let records = btree_v2::collect_btree_v2_records_storage(
487            self.context.storage.as_ref(),
488            &btree_header,
489            self.offset_size(),
490            self.length_size(),
491            None,
492            &[],
493            None,
494        )?;
495
496        let mut children = Vec::new();
497        let mut direct_block_cache = FractalHeapDirectBlockCache::default();
498        for record in &records {
499            let heap_id = match record {
500                btree_v2::BTreeV2Record::LinkNameHash { heap_id, .. }
501                | btree_v2::BTreeV2Record::CreationOrder { heap_id, .. } => heap_id,
502                _ => continue,
503            };
504
505            let managed_bytes = heap.get_object_storage_cached_with_registry(
506                heap_id,
507                self.context.storage.as_ref(),
508                self.offset_size(),
509                self.length_size(),
510                &mut direct_block_cache,
511                Some(self.context.filter_registry.as_ref()),
512            )?;
513
514            let mut link_cursor = Cursor::new(&managed_bytes);
515            let link_msg = link::parse(
516                &mut link_cursor,
517                self.offset_size(),
518                self.length_size(),
519                managed_bytes.len(),
520            )?;
521
522            match &link_msg.target {
523                LinkTarget::Hard { address } => {
524                    children.push(ChildEntry {
525                        name: link_msg.name.clone(),
526                        location: self.local_location(*address),
527                    });
528                }
529                LinkTarget::Soft { path } => {
530                    if let Ok(location) = self.resolve_soft_link_depth(path, link_depth) {
531                        children.push(ChildEntry {
532                            name: link_msg.name.clone(),
533                            location,
534                        });
535                    }
536                }
537                LinkTarget::External { filename, path } => {
538                    if let Some(location) =
539                        self.resolve_external_link_depth(filename, path, link_depth)?
540                    {
541                        children.push(ChildEntry {
542                            name: link_msg.name.clone(),
543                            location,
544                        });
545                    }
546                }
547            }
548        }
549
550        Ok(children)
551    }
552
553    fn resolve_dense_link_storage(
554        &self,
555        link_info: &LinkInfoMessage,
556        name: &str,
557        link_depth: u32,
558    ) -> Result<Option<ChildEntry>> {
559        let heap = FractalHeap::parse_at_storage(
560            self.context.storage.as_ref(),
561            link_info.fractal_heap_address,
562            self.offset_size(),
563            self.length_size(),
564        )?;
565
566        let btree_header = btree_v2::BTreeV2Header::parse_at_storage(
567            self.context.storage.as_ref(),
568            link_info.btree_name_index_address,
569            self.offset_size(),
570            self.length_size(),
571        )?;
572
573        let records = btree_v2::collect_btree_v2_link_name_hash_records_storage(
574            self.context.storage.as_ref(),
575            &btree_header,
576            self.offset_size(),
577            self.length_size(),
578            jenkins_lookup3(name.as_bytes()),
579        )?;
580
581        let mut direct_block_cache = FractalHeapDirectBlockCache::default();
582        for record in &records {
583            let btree_v2::BTreeV2Record::LinkNameHash { heap_id, .. } = record else {
584                continue;
585            };
586
587            let managed_bytes = heap.get_object_storage_cached_with_registry(
588                heap_id,
589                self.context.storage.as_ref(),
590                self.offset_size(),
591                self.length_size(),
592                &mut direct_block_cache,
593                Some(self.context.filter_registry.as_ref()),
594            )?;
595
596            let mut link_cursor = Cursor::new(&managed_bytes);
597            let link_msg = link::parse(
598                &mut link_cursor,
599                self.offset_size(),
600                self.length_size(),
601                managed_bytes.len(),
602            )?;
603            if link_msg.name == name {
604                return self.resolve_link_message_target(&link_msg, link_depth);
605            }
606        }
607
608        Ok(None)
609    }
610
611    pub fn child_name_by_address(&self, address: u64) -> Result<Option<String>> {
612        Ok(self
613            .resolve_children()?
614            .into_iter()
615            .find(|child| child.location.address == address)
616            .map(|child| child.name))
617    }
618
619    fn child_context(&self, child: &ChildEntry) -> String {
620        format!("child '{}' at {:#x}", child.name, child.location.address)
621    }
622
623    fn child_object_kind(&self, child: &ChildEntry) -> Result<ChildObjectKind> {
624        let header = self
625            .cached_child_header(child)
626            .map_err(|err| err.with_context(self.child_context(child)))?;
627
628        Ok(classify_child_header(header.as_ref()))
629    }
630
631    fn try_open_child_dataset(&self, child: &ChildEntry) -> Result<Option<Dataset>> {
632        let header = self
633            .cached_child_header(child)
634            .map_err(|err| err.with_context(self.child_context(child)))?;
635
636        if classify_child_header(header.as_ref()) != ChildObjectKind::Dataset {
637            return Ok(None);
638        }
639
640        Dataset::from_parsed_header(
641            crate::dataset::DatasetParseContext {
642                context: child.location.context.clone(),
643            },
644            child.location.address,
645            child.name.clone(),
646            header.as_ref(),
647        )
648        .map(Some)
649        .map_err(|err| err.with_context(self.child_context(child)))
650    }
651
652    fn cached_child_header(
653        &self,
654        child: &ChildEntry,
655    ) -> Result<Arc<crate::object_header::ObjectHeader>> {
656        child
657            .location
658            .context
659            .get_or_parse_header(child.location.address)
660    }
661
662    /// Maximum nesting depth for soft link resolution.
663    const MAX_SOFT_LINK_DEPTH: u32 = 16;
664
665    fn resolve_soft_link_depth(&self, path: &str, depth: u32) -> Result<ObjectLocation> {
666        self.resolve_path_location(path, depth, "soft link")
667    }
668
669    fn resolve_external_link_depth(
670        &self,
671        filename: &str,
672        path: &str,
673        depth: u32,
674    ) -> Result<Option<ObjectLocation>> {
675        if depth >= Self::MAX_SOFT_LINK_DEPTH {
676            return Err(Error::Other(format!(
677                "external link resolution exceeded maximum depth ({}) at '{}:{}'",
678                Self::MAX_SOFT_LINK_DEPTH,
679                filename,
680                path,
681            )));
682        }
683
684        let Some(resolver) = self.context.external_link_resolver.as_ref() else {
685            return Ok(None);
686        };
687        let Some(file) = resolver.resolve_external_link(filename)? else {
688            return Ok(None);
689        };
690        let root = file.root_group()?;
691        Ok(Some(root.resolve_path_location(
692            path,
693            depth + 1,
694            "external link",
695        )?))
696    }
697
698    fn resolve_path_location(
699        &self,
700        path: &str,
701        depth: u32,
702        link_kind: &str,
703    ) -> Result<ObjectLocation> {
704        if depth >= Self::MAX_SOFT_LINK_DEPTH {
705            return Err(Error::Other(format!(
706                "{} resolution exceeded maximum depth ({}) — possible cycle at '{}'",
707                link_kind,
708                Self::MAX_SOFT_LINK_DEPTH,
709                path,
710            )));
711        }
712
713        let parts: Vec<&str> = path
714            .trim_matches('/')
715            .split('/')
716            .filter(|s| !s.is_empty())
717            .collect();
718
719        if parts.is_empty() {
720            return Ok(self.local_location(self.root_address));
721        }
722
723        let start_addr = if path.starts_with('/') {
724            self.root_address
725        } else {
726            self.address
727        };
728
729        let mut current_group = Group::new(
730            self.context.clone(),
731            start_addr,
732            String::new(),
733            self.root_address,
734        );
735
736        for &part in &parts[..parts.len() - 1] {
737            current_group = current_group.group(part)?;
738        }
739
740        let target_name = parts[parts.len() - 1];
741        if let Some(child) = current_group.resolve_child_with_link_depth(target_name, depth + 1)? {
742            return Ok(child.location);
743        }
744
745        Err(Error::Other(format!(
746            "{} target '{}' not found",
747            link_kind, path
748        )))
749    }
750}
751
752fn classify_child_header(header: &crate::object_header::ObjectHeader) -> ChildObjectKind {
753    let mut has_dataset_message = false;
754    let mut has_attribute_message = false;
755
756    for msg in &header.messages {
757        match msg {
758            HdfMessage::SymbolTable(_)
759            | HdfMessage::Link(_)
760            | HdfMessage::LinkInfo(_)
761            | HdfMessage::GroupInfo(_) => return ChildObjectKind::Group,
762            HdfMessage::Attribute(_) | HdfMessage::AttributeInfo(_) => {
763                has_attribute_message = true;
764            }
765            HdfMessage::Dataspace(_)
766            | HdfMessage::DataLayout(_)
767            | HdfMessage::FillValue(_)
768            | HdfMessage::FilterPipeline(_) => has_dataset_message = true,
769            _ => {}
770        }
771    }
772
773    if has_dataset_message {
774        ChildObjectKind::Dataset
775    } else if has_attribute_message {
776        ChildObjectKind::Group
777    } else {
778        ChildObjectKind::Other
779    }
780}
hdf5_reader/group.rs

hdf5_reader/
group.rs
