hdf5_reader/
group.rs

1use std::sync::Arc;
2
3use crate::attribute_api::{
4    collect_attribute_messages_storage, decoded_vlen_strings_storage, resolve_vlen_bytes_storage,
5    Attribute,
6};
7use crate::btree_v1;
8use crate::btree_v2;
9use crate::checksum::jenkins_lookup3;
10use crate::dataset::Dataset;
11use crate::error::{Error, Result};
12use crate::fractal_heap::{FractalHeap, FractalHeapDirectBlockCache};
13use crate::io::Cursor;
14use crate::local_heap::LocalHeap;
15use crate::messages::datatype::VarLenKind;
16use crate::messages::link::{self, LinkMessage, LinkTarget};
17use crate::messages::link_info::LinkInfoMessage;
18use crate::messages::symbol_table_msg::SymbolTableMessage;
19use crate::messages::HdfMessage;
20use crate::storage::Storage;
21use crate::FileContext;
22
23/// A group within an HDF5 file.
24#[derive(Clone)]
25pub struct Group {
26    context: Arc<FileContext>,
27    pub(crate) name: String,
28    pub(crate) address: u64,
29    /// Address of the root group's object header, used for resolving soft links.
30    pub(crate) root_address: u64,
31}
32
33#[derive(Clone)]
34struct ChildEntry {
35    name: String,
36    location: ObjectLocation,
37}
38
39#[derive(Clone)]
40struct ObjectLocation {
41    context: Arc<FileContext>,
42    address: u64,
43    root_address: u64,
44}
45
46#[derive(Debug, Clone, Copy, PartialEq, Eq)]
47enum ChildObjectKind {
48    Group,
49    Dataset,
50    Other,
51}
52
53impl Group {
54    /// Create a group from a known object header address.
55    pub(crate) fn new(
56        context: Arc<FileContext>,
57        address: u64,
58        name: String,
59        root_address: u64,
60    ) -> Self {
61        Group {
62            context,
63            name,
64            address,
65            root_address,
66        }
67    }
68
69    /// Group name.
70    pub fn name(&self) -> &str {
71        &self.name
72    }
73
74    /// Object header address of this group within the file.
75    pub fn address(&self) -> u64 {
76        self.address
77    }
78
79    /// Materialize the full file backing this group.
80    pub fn file_data(&self) -> Result<crate::storage::StorageBuffer> {
81        self.context.full_file_data()
82    }
83
84    /// Access the underlying random-access storage backend.
85    pub fn storage(&self) -> &dyn Storage {
86        self.context.storage.as_ref()
87    }
88
89    /// Size of file offsets in bytes.
90    pub fn offset_size(&self) -> u8 {
91        self.context.superblock.offset_size
92    }
93
94    /// Size of file lengths in bytes.
95    pub fn length_size(&self) -> u8 {
96        self.context.superblock.length_size
97    }
98
99    /// Parse (or retrieve from cache) the object header at the given address.
100    fn cached_header(&self, addr: u64) -> Result<Arc<crate::object_header::ObjectHeader>> {
101        self.context.get_or_parse_header(addr)
102    }
103
104    fn local_location(&self, address: u64) -> ObjectLocation {
105        ObjectLocation {
106            context: self.context.clone(),
107            address,
108            root_address: self.root_address,
109        }
110    }
111
112    /// List all child groups.
113    pub fn groups(&self) -> Result<Vec<Group>> {
114        let (groups, _) = self.resolve_member_objects()?;
115        Ok(groups)
116    }
117
118    /// List all child members, partitioned into groups and datasets.
119    pub fn members(&self) -> Result<(Vec<Group>, Vec<Dataset>)> {
120        self.resolve_member_objects()
121    }
122
123    fn resolve_member_objects(&self) -> Result<(Vec<Group>, Vec<Dataset>)> {
124        let children = self.resolve_children()?;
125        let mut groups = Vec::new();
126        let mut datasets = Vec::new();
127        for child in &children {
128            match self.child_object_kind(child)? {
129                ChildObjectKind::Group => {
130                    groups.push(Group::new(
131                        child.location.context.clone(),
132                        child.location.address,
133                        child.name.clone(),
134                        child.location.root_address,
135                    ));
136                }
137                ChildObjectKind::Dataset => {
138                    if let Some(dataset) = self.try_open_child_dataset(child)? {
139                        datasets.push(dataset);
140                    }
141                }
142                ChildObjectKind::Other => {}
143            }
144        }
145        Ok((groups, datasets))
146    }
147
148    /// Get a child group by name.
149    pub fn group(&self, name: &str) -> Result<Group> {
150        if let Some(child) = self.resolve_child(name)? {
151            return match self.child_object_kind(&child)? {
152                ChildObjectKind::Group => Ok(Group::new(
153                    child.location.context.clone(),
154                    child.location.address,
155                    child.name.clone(),
156                    child.location.root_address,
157                )),
158                ChildObjectKind::Dataset => Err(Error::GroupNotFound(format!(
159                    "'{}' is a dataset, not a group",
160                    name
161                ))),
162                ChildObjectKind::Other => {
163                    Err(Error::GroupNotFound(format!("'{}' is not a group", name)))
164                }
165            };
166        }
167        Err(Error::GroupNotFound(name.to_string()))
168    }
169
170    /// List all child datasets.
171    pub fn datasets(&self) -> Result<Vec<Dataset>> {
172        let (_, datasets) = self.resolve_member_objects()?;
173        Ok(datasets)
174    }
175
176    /// Get a child dataset by name.
177    pub fn dataset(&self, name: &str) -> Result<Dataset> {
178        if let Some(child) = self.resolve_child(name)? {
179            if let Some(dataset) = self.try_open_child_dataset(&child)? {
180                return Ok(dataset);
181            }
182            return Err(Error::DatasetNotFound(name.to_string()));
183        }
184        Err(Error::DatasetNotFound(name.to_string()))
185    }
186
187    /// List attributes on this group.
188    pub fn attributes(&self) -> Result<Vec<Attribute>> {
189        // `cached_header` already resolves shared messages, including those
190        // stored in the shared object header message (SOHM) table, so the
191        // group attribute path supports the same sharing the dataset path does.
192        let header = self.cached_header(self.address)?;
193        Ok(collect_attribute_messages_storage(
194            &header,
195            self.context.storage.as_ref(),
196            self.offset_size(),
197            self.length_size(),
198            Some(self.context.filter_registry.as_ref()),
199        )?
200        .into_iter()
201        .map(|attr| {
202            let element_count = attr.dataspace.num_elements().ok();
203            let decoded_strings = element_count.and_then(|element_count| {
204                decoded_vlen_strings_storage(
205                    &attr.datatype,
206                    &attr.raw_data,
207                    self.context.storage.as_ref(),
208                    self.offset_size(),
209                    self.length_size(),
210                    element_count,
211                )
212            });
213            let raw_data = match &attr.datatype {
214                crate::messages::datatype::Datatype::VarLen {
215                    base,
216                    kind: VarLenKind::String,
217                    ..
218                } if matches!(
219                    base.as_ref(),
220                    crate::messages::datatype::Datatype::FixedPoint { size: 1, .. }
221                ) && matches!(attr.dataspace.num_elements(), Ok(1)) =>
222                {
223                    resolve_vlen_bytes_storage(
224                        &attr.raw_data,
225                        self.context.storage.as_ref(),
226                        self.offset_size(),
227                        self.length_size(),
228                    )
229                    .unwrap_or_else(|| attr.raw_data.clone())
230                }
231                _ => attr.raw_data.clone(),
232            };
233            Attribute {
234                name: attr.name,
235                datatype: attr.datatype,
236                shape: match attr.dataspace.dataspace_type {
237                    crate::messages::dataspace::DataspaceType::Scalar => vec![],
238                    crate::messages::dataspace::DataspaceType::Null => vec![0],
239                    crate::messages::dataspace::DataspaceType::Simple => attr.dataspace.dims,
240                },
241                raw_data,
242                decoded_strings,
243            }
244        })
245        .collect())
246    }
247
248    /// Find an attribute by name.
249    pub fn attribute(&self, name: &str) -> Result<Attribute> {
250        let attrs = self.attributes()?;
251        attrs
252            .into_iter()
253            .find(|a| a.name == name)
254            .ok_or_else(|| Error::AttributeNotFound(name.to_string()))
255    }
256
257    /// Resolve children from the object header.
258    /// Handles both old-style (symbol table) and new-style (link messages) groups.
259    fn resolve_children(&self) -> Result<Vec<ChildEntry>> {
260        self.resolve_children_with_link_depth(0)
261    }
262
263    fn resolve_child(&self, name: &str) -> Result<Option<ChildEntry>> {
264        self.resolve_child_with_link_depth(name, 0)
265    }
266
267    fn resolve_child_with_link_depth(
268        &self,
269        name: &str,
270        link_depth: u32,
271    ) -> Result<Option<ChildEntry>> {
272        let header = self.cached_header(self.address)?;
273
274        let mut link_info: Option<LinkInfoMessage> = None;
275        let mut matching_compact_link: Option<LinkMessage> = None;
276
277        for msg in &header.messages {
278            match msg {
279                HdfMessage::SymbolTable(st) => {
280                    return Ok(self
281                        .resolve_old_style_group_storage(st)?
282                        .into_iter()
283                        .find(|child| child.name == name));
284                }
285                HdfMessage::Link(link) if link.name == name => {
286                    matching_compact_link = Some(link.clone());
287                }
288                HdfMessage::LinkInfo(li) => {
289                    link_info = Some(li.clone());
290                }
291                _ => {}
292            }
293        }
294
295        if let Some(link) = matching_compact_link {
296            if let Some(child) = self.resolve_link_message_target(&link, link_depth)? {
297                return Ok(Some(child));
298            }
299        }
300
301        if let Some(ref li) = link_info {
302            if !Cursor::is_undefined_offset(li.fractal_heap_address, self.offset_size()) {
303                return self.resolve_dense_link_storage(li, name, link_depth);
304            }
305        }
306
307        Ok(None)
308    }
309
310    /// Resolve children with a soft-link depth counter to prevent cycles.
311    fn resolve_children_with_link_depth(&self, link_depth: u32) -> Result<Vec<ChildEntry>> {
312        let header = self.cached_header(self.address)?;
313
314        let mut children = Vec::new();
315
316        // Check for old-style groups (symbol table message)
317        let mut found_symbol_table = false;
318        // Check for new-style groups (link messages)
319        let mut link_info: Option<LinkInfoMessage> = None;
320        let mut links: Vec<LinkMessage> = Vec::new();
321
322        for msg in &header.messages {
323            match msg {
324                HdfMessage::SymbolTable(st) => {
325                    found_symbol_table = true;
326                    children = self.resolve_old_style_group_storage(st)?;
327                }
328                HdfMessage::Link(link) => {
329                    links.push(link.clone());
330                }
331                HdfMessage::LinkInfo(li) => {
332                    link_info = Some(li.clone());
333                }
334                _ => {}
335            }
336        }
337
338        if !found_symbol_table {
339            // New-style group: use compact links from header messages
340            self.resolve_link_targets(&links, link_depth, &mut children)?;
341
342            // Dense-link storage can coexist with compact links, so merge both.
343            if let Some(ref li) = link_info {
344                if !Cursor::is_undefined_offset(li.fractal_heap_address, self.offset_size()) {
345                    for child in self.resolve_dense_links_storage(li, link_depth)? {
346                        let is_duplicate = children.iter().any(|existing| {
347                            existing.name == child.name
348                                && existing.location.address == child.location.address
349                                && Arc::ptr_eq(&existing.location.context, &child.location.context)
350                        });
351                        if !is_duplicate {
352                            children.push(child);
353                        }
354                    }
355                }
356            }
357        }
358
359        Ok(children)
360    }
361
362    /// Resolve link targets (hard and soft), appending to `children`.
363    fn resolve_link_targets(
364        &self,
365        links: &[LinkMessage],
366        link_depth: u32,
367        children: &mut Vec<ChildEntry>,
368    ) -> Result<()> {
369        for link in links {
370            if let Some(child) = self.resolve_link_message_target(link, link_depth)? {
371                children.push(child);
372            }
373        }
374        Ok(())
375    }
376
377    fn resolve_link_message_target(
378        &self,
379        link: &LinkMessage,
380        link_depth: u32,
381    ) -> Result<Option<ChildEntry>> {
382        match &link.target {
383            LinkTarget::Hard { address } => Ok(Some(ChildEntry {
384                name: link.name.clone(),
385                location: self.local_location(*address),
386            })),
387            LinkTarget::Soft { path } => Ok(self
388                .resolve_soft_link_depth(path, link_depth)
389                .ok()
390                .map(|location| ChildEntry {
391                    name: link.name.clone(),
392                    location,
393                })),
394            LinkTarget::External { filename, path } => Ok(self
395                .resolve_external_link_depth(filename, path, link_depth)?
396                .map(|location| ChildEntry {
397                    name: link.name.clone(),
398                    location,
399                })),
400        }
401    }
402
403    fn resolve_old_style_group_storage(&self, st: &SymbolTableMessage) -> Result<Vec<ChildEntry>> {
404        let heap = LocalHeap::parse_at_storage(
405            self.context.storage.as_ref(),
406            st.heap_address,
407            self.offset_size(),
408            self.length_size(),
409        )?;
410
411        let leaves = btree_v1::collect_btree_v1_leaves_storage(
412            self.context.storage.as_ref(),
413            st.btree_address,
414            self.offset_size(),
415            self.length_size(),
416            None,
417            &[],
418            None,
419        )?;
420
421        let mut children = Vec::new();
422        for (_key, snod_address) in &leaves {
423            let header_len = 8 + 2 * usize::from(self.offset_size());
424            let prefix = self.context.read_range(*snod_address, header_len)?;
425            let mut prefix_cursor = Cursor::new(prefix.as_ref());
426            let sig = prefix_cursor.read_bytes(4)?;
427            if sig != *b"SNOD" {
428                return Err(Error::InvalidData(format!(
429                    "expected SNOD signature at offset {:#x}",
430                    snod_address
431                )));
432            }
433            let version = prefix_cursor.read_u8()?;
434            if version != 1 {
435                return Err(Error::InvalidData(format!(
436                    "unsupported symbol table node version {}",
437                    version
438                )));
439            }
440            prefix_cursor.skip(1)?;
441            let num_symbols = prefix_cursor.read_u16_le()?;
442            let node_len =
443                8 + usize::from(num_symbols) * (2 * usize::from(self.offset_size()) + 4 + 4 + 16);
444            let bytes = self.context.read_range(*snod_address, node_len)?;
445            let mut cursor = Cursor::new(bytes.as_ref());
446            let snod = crate::symbol_table::SymbolTableNode::parse(
447                &mut cursor,
448                self.offset_size(),
449                self.length_size(),
450            )?;
451
452            for entry in &snod.entries {
453                let name =
454                    heap.get_string_storage(entry.link_name_offset, self.context.storage.as_ref())?;
455                children.push(ChildEntry {
456                    name,
457                    location: self.local_location(entry.object_header_address),
458                });
459            }
460        }
461
462        Ok(children)
463    }
464
465    fn resolve_dense_links_storage(
466        &self,
467        link_info: &LinkInfoMessage,
468        link_depth: u32,
469    ) -> Result<Vec<ChildEntry>> {
470        let heap = FractalHeap::parse_at_storage(
471            self.context.storage.as_ref(),
472            link_info.fractal_heap_address,
473            self.offset_size(),
474            self.length_size(),
475        )?;
476
477        let btree_header = btree_v2::BTreeV2Header::parse_at_storage(
478            self.context.storage.as_ref(),
479            link_info.btree_name_index_address,
480            self.offset_size(),
481            self.length_size(),
482        )?;
483
484        let records = btree_v2::collect_btree_v2_records_storage(
485            self.context.storage.as_ref(),
486            &btree_header,
487            self.offset_size(),
488            self.length_size(),
489            None,
490            &[],
491            None,
492        )?;
493
494        let mut children = Vec::new();
495        let mut direct_block_cache = FractalHeapDirectBlockCache::default();
496        for record in &records {
497            let heap_id = match record {
498                btree_v2::BTreeV2Record::LinkNameHash { heap_id, .. }
499                | btree_v2::BTreeV2Record::CreationOrder { heap_id, .. } => heap_id,
500                _ => continue,
501            };
502
503            let managed_bytes = heap.get_object_storage_cached_with_registry(
504                heap_id,
505                self.context.storage.as_ref(),
506                self.offset_size(),
507                self.length_size(),
508                &mut direct_block_cache,
509                Some(self.context.filter_registry.as_ref()),
510            )?;
511
512            let mut link_cursor = Cursor::new(&managed_bytes);
513            let link_msg = link::parse(
514                &mut link_cursor,
515                self.offset_size(),
516                self.length_size(),
517                managed_bytes.len(),
518            )?;
519
520            match &link_msg.target {
521                LinkTarget::Hard { address } => {
522                    children.push(ChildEntry {
523                        name: link_msg.name.clone(),
524                        location: self.local_location(*address),
525                    });
526                }
527                LinkTarget::Soft { path } => {
528                    if let Ok(location) = self.resolve_soft_link_depth(path, link_depth) {
529                        children.push(ChildEntry {
530                            name: link_msg.name.clone(),
531                            location,
532                        });
533                    }
534                }
535                LinkTarget::External { filename, path } => {
536                    if let Some(location) =
537                        self.resolve_external_link_depth(filename, path, link_depth)?
538                    {
539                        children.push(ChildEntry {
540                            name: link_msg.name.clone(),
541                            location,
542                        });
543                    }
544                }
545            }
546        }
547
548        Ok(children)
549    }
550
551    fn resolve_dense_link_storage(
552        &self,
553        link_info: &LinkInfoMessage,
554        name: &str,
555        link_depth: u32,
556    ) -> Result<Option<ChildEntry>> {
557        let heap = FractalHeap::parse_at_storage(
558            self.context.storage.as_ref(),
559            link_info.fractal_heap_address,
560            self.offset_size(),
561            self.length_size(),
562        )?;
563
564        let btree_header = btree_v2::BTreeV2Header::parse_at_storage(
565            self.context.storage.as_ref(),
566            link_info.btree_name_index_address,
567            self.offset_size(),
568            self.length_size(),
569        )?;
570
571        let records = btree_v2::collect_btree_v2_link_name_hash_records_storage(
572            self.context.storage.as_ref(),
573            &btree_header,
574            self.offset_size(),
575            self.length_size(),
576            jenkins_lookup3(name.as_bytes()),
577        )?;
578
579        let mut direct_block_cache = FractalHeapDirectBlockCache::default();
580        for record in &records {
581            let btree_v2::BTreeV2Record::LinkNameHash { heap_id, .. } = record else {
582                continue;
583            };
584
585            let managed_bytes = heap.get_object_storage_cached_with_registry(
586                heap_id,
587                self.context.storage.as_ref(),
588                self.offset_size(),
589                self.length_size(),
590                &mut direct_block_cache,
591                Some(self.context.filter_registry.as_ref()),
592            )?;
593
594            let mut link_cursor = Cursor::new(&managed_bytes);
595            let link_msg = link::parse(
596                &mut link_cursor,
597                self.offset_size(),
598                self.length_size(),
599                managed_bytes.len(),
600            )?;
601            if link_msg.name == name {
602                return self.resolve_link_message_target(&link_msg, link_depth);
603            }
604        }
605
606        Ok(None)
607    }
608
609    pub fn child_name_by_address(&self, address: u64) -> Result<Option<String>> {
610        Ok(self
611            .resolve_children()?
612            .into_iter()
613            .find(|child| child.location.address == address)
614            .map(|child| child.name))
615    }
616
617    fn child_context(&self, child: &ChildEntry) -> String {
618        format!("child '{}' at {:#x}", child.name, child.location.address)
619    }
620
621    fn child_object_kind(&self, child: &ChildEntry) -> Result<ChildObjectKind> {
622        let header = self
623            .cached_child_header(child)
624            .map_err(|err| err.with_context(self.child_context(child)))?;
625
626        Ok(classify_child_header(header.as_ref()))
627    }
628
629    fn try_open_child_dataset(&self, child: &ChildEntry) -> Result<Option<Dataset>> {
630        let header = self
631            .cached_child_header(child)
632            .map_err(|err| err.with_context(self.child_context(child)))?;
633
634        if classify_child_header(header.as_ref()) != ChildObjectKind::Dataset {
635            return Ok(None);
636        }
637
638        Dataset::from_parsed_header(
639            crate::dataset::DatasetParseContext {
640                context: child.location.context.clone(),
641            },
642            child.location.address,
643            child.name.clone(),
644            header.as_ref(),
645        )
646        .map(Some)
647        .map_err(|err| err.with_context(self.child_context(child)))
648    }
649
650    fn cached_child_header(
651        &self,
652        child: &ChildEntry,
653    ) -> Result<Arc<crate::object_header::ObjectHeader>> {
654        child
655            .location
656            .context
657            .get_or_parse_header(child.location.address)
658    }
659
660    /// Maximum nesting depth for soft link resolution.
661    const MAX_SOFT_LINK_DEPTH: u32 = 16;
662
663    fn resolve_soft_link_depth(&self, path: &str, depth: u32) -> Result<ObjectLocation> {
664        self.resolve_path_location(path, depth, "soft link")
665    }
666
667    fn resolve_external_link_depth(
668        &self,
669        filename: &str,
670        path: &str,
671        depth: u32,
672    ) -> Result<Option<ObjectLocation>> {
673        if depth >= Self::MAX_SOFT_LINK_DEPTH {
674            return Err(Error::Other(format!(
675                "external link resolution exceeded maximum depth ({}) at '{}:{}'",
676                Self::MAX_SOFT_LINK_DEPTH,
677                filename,
678                path,
679            )));
680        }
681
682        let Some(resolver) = self.context.external_link_resolver.as_ref() else {
683            return Ok(None);
684        };
685        let Some(file) = resolver.resolve_external_link(filename)? else {
686            return Ok(None);
687        };
688        let root = file.root_group()?;
689        Ok(Some(root.resolve_path_location(
690            path,
691            depth + 1,
692            "external link",
693        )?))
694    }
695
696    fn resolve_path_location(
697        &self,
698        path: &str,
699        depth: u32,
700        link_kind: &str,
701    ) -> Result<ObjectLocation> {
702        if depth >= Self::MAX_SOFT_LINK_DEPTH {
703            return Err(Error::Other(format!(
704                "{} resolution exceeded maximum depth ({}) — possible cycle at '{}'",
705                link_kind,
706                Self::MAX_SOFT_LINK_DEPTH,
707                path,
708            )));
709        }
710
711        let parts: Vec<&str> = path
712            .trim_matches('/')
713            .split('/')
714            .filter(|s| !s.is_empty())
715            .collect();
716
717        if parts.is_empty() {
718            return Ok(self.local_location(self.root_address));
719        }
720
721        let start_addr = if path.starts_with('/') {
722            self.root_address
723        } else {
724            self.address
725        };
726
727        let mut current_group = Group::new(
728            self.context.clone(),
729            start_addr,
730            String::new(),
731            self.root_address,
732        );
733
734        for &part in &parts[..parts.len() - 1] {
735            current_group = current_group.group(part)?;
736        }
737
738        let target_name = parts[parts.len() - 1];
739        if let Some(child) = current_group.resolve_child_with_link_depth(target_name, depth + 1)? {
740            return Ok(child.location);
741        }
742
743        Err(Error::Other(format!(
744            "{} target '{}' not found",
745            link_kind, path
746        )))
747    }
748}
749
750fn classify_child_header(header: &crate::object_header::ObjectHeader) -> ChildObjectKind {
751    let mut has_dataset_message = false;
752    let mut has_attribute_message = false;
753
754    for msg in &header.messages {
755        match msg {
756            HdfMessage::SymbolTable(_)
757            | HdfMessage::Link(_)
758            | HdfMessage::LinkInfo(_)
759            | HdfMessage::GroupInfo(_) => return ChildObjectKind::Group,
760            HdfMessage::Attribute(_) | HdfMessage::AttributeInfo(_) => {
761                has_attribute_message = true;
762            }
763            HdfMessage::Dataspace(_)
764            | HdfMessage::DataLayout(_)
765            | HdfMessage::FillValue(_)
766            | HdfMessage::FilterPipeline(_) => has_dataset_message = true,
767            _ => {}
768        }
769    }
770
771    if has_dataset_message {
772        ChildObjectKind::Dataset
773    } else if has_attribute_message {
774        ChildObjectKind::Group
775    } else {
776        ChildObjectKind::Other
777    }
778}
hdf5_reader/group.rs

hdf5_reader/
group.rs
