use crate::format::btree_v1::BTreeV1Node;
use crate::format::free_space::FreeSpaceClass;
use crate::format::local_heap::{
local_heap_get_string, local_heap_header_size, LocalHeapHeader, LocalHeapImage,
LOCAL_HEAP_FREE_NULL,
};
use crate::format::superblock::{SymbolTableCache, SymbolTableEntry};
use crate::format::symbol_table::SymbolTableNode;
use crate::format::{FormatContext, FormatError, UNDEF_ADDR};
use crate::io::allocator::FileAllocator;
use crate::io::file_handle::FileHandle;
use crate::io::{FileMeta, IoError, IoResult};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct Stab {
pub btree_addr: u64,
pub heap_addr: u64,
}
impl Stab {
pub(crate) fn encode(&self, ctx: &FormatContext) -> Vec<u8> {
let sa = ctx.sizeof_addr as usize;
let mut buf = Vec::with_capacity(2 * sa);
buf.extend_from_slice(&self.btree_addr.to_le_bytes()[..sa]);
buf.extend_from_slice(&self.heap_addr.to_le_bytes()[..sa]);
buf
}
pub(crate) fn decode(data: &[u8], ctx: &FormatContext) -> Option<Self> {
let sa = ctx.sizeof_addr as usize;
if data.len() < 2 * sa {
return None;
}
Some(Self {
btree_addr: crate::format::bytes::read_le_addr(data, sa),
heap_addr: crate::format::bytes::read_le_addr(&data[sa..], sa),
})
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum StabTarget {
Hard { addr: u64, cached: Option<Stab> },
Soft { value: String },
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct StabLink {
pub name: String,
pub target: StabTarget,
}
#[derive(Debug, Default, Clone)]
pub(crate) struct StabExtents {
pub blocks: Vec<(u64, u64)>,
}
#[derive(Debug, Default)]
pub(crate) struct StabContents {
pub links: Vec<StabLink>,
pub extents: StabExtents,
}
const MAX_BTREE_DEPTH: usize = 256;
pub(crate) fn read_stab(
handle: &FileHandle,
meta: &FileMeta,
stab: Stab,
) -> IoResult<StabContents> {
let sa = meta.ctx.sizeof_addr as usize;
let ss = meta.ctx.sizeof_size as usize;
let mut out = StabContents::default();
let heap_hdr_buf = handle.read_at_most(stab.heap_addr, 64)?;
let heap_hdr = LocalHeapHeader::decode(&heap_hdr_buf, sa, ss)?;
let heap_data = handle.read_at(heap_hdr.data_addr, heap_hdr.data_size as usize)?;
out.extents
.blocks
.push((stab.heap_addr, local_heap_header_size(sa, ss) as u64));
out.extents
.blocks
.push((heap_hdr.data_addr, heap_hdr.data_size));
let snod_size = meta.btree.symbol_table_node_size(sa, ss);
let mut visited = std::collections::HashSet::new();
let mut snod_addrs = Vec::new();
collect_snods(
handle,
meta,
stab.btree_addr,
0,
&mut visited,
&mut snod_addrs,
&mut out.extents,
)?;
for snod_addr in snod_addrs {
out.extents.blocks.push((snod_addr, snod_size as u64));
let buf = handle.read_at_most(snod_addr, snod_size)?;
let snod = SymbolTableNode::decode(&buf, sa, ss, meta.btree.sym_leaf_max_entries())?;
for entry in &snod.entries {
let name = local_heap_get_string(&heap_data, entry.name_offset)?;
if name.is_empty() {
continue;
}
let target = match entry.cache {
SymbolTableCache::SoftLink { value_offset } => StabTarget::Soft {
value: local_heap_get_string(&heap_data, value_offset as u64)?,
},
SymbolTableCache::SymbolTable {
btree_addr,
heap_addr,
} => StabTarget::Hard {
addr: entry.obj_header_addr,
cached: Some(Stab {
btree_addr,
heap_addr,
}),
},
SymbolTableCache::Nothing => StabTarget::Hard {
addr: entry.obj_header_addr,
cached: None,
},
};
out.links.push(StabLink { name, target });
}
}
Ok(out)
}
fn collect_snods(
handle: &FileHandle,
meta: &FileMeta,
tree_addr: u64,
depth: usize,
visited: &mut std::collections::HashSet<u64>,
out: &mut Vec<u64>,
extents: &mut StabExtents,
) -> IoResult<()> {
if depth > MAX_BTREE_DEPTH || !visited.insert(tree_addr) {
return Ok(());
}
let sa = meta.ctx.sizeof_addr as usize;
let ss = meta.ctx.sizeof_size as usize;
let node_size = meta.btree.snode_btree_node_size(sa, ss);
let buf = handle.read_at_most(tree_addr, node_size)?;
let node = BTreeV1Node::decode(&buf, sa, ss, meta.btree.snode_max_entries())?;
extents.blocks.push((tree_addr, node_size as u64));
if node.level == 0 {
out.extend_from_slice(&node.children);
return Ok(());
}
for &child in &node.children {
collect_snods(handle, meta, child, depth + 1, visited, out, extents)?;
}
Ok(())
}
pub(crate) fn write_stab(
handle: &FileHandle,
allocator: &FileAllocator,
meta: &FileMeta,
links: &[StabLink],
) -> IoResult<Stab> {
let sa = meta.ctx.sizeof_addr as usize;
let ss = meta.ctx.sizeof_size as usize;
let cfg = &meta.btree;
let mut heap = LocalHeapImage::with_empty_string();
let mut entries: Vec<SymbolTableEntry> = Vec::with_capacity(links.len());
let mut names: Vec<&str> = Vec::with_capacity(links.len());
for link in links {
let name_offset = heap.insert_str(&link.name);
let (obj_header_addr, cache) = match &link.target {
StabTarget::Hard { addr, cached } => (
*addr,
match cached {
Some(s) => SymbolTableCache::SymbolTable {
btree_addr: s.btree_addr,
heap_addr: s.heap_addr,
},
None => SymbolTableCache::Nothing,
},
),
StabTarget::Soft { value } => {
let value_offset = heap.insert_str(value);
let Ok(value_offset) = u32::try_from(value_offset) else {
return Err(IoError::Format(FormatError::InvalidData(format!(
"the soft link '{}' lands at heap offset {value_offset}, past the \
4-byte offset a symbol table entry's scratch pad can hold",
link.name
))));
};
(UNDEF_ADDR, SymbolTableCache::SoftLink { value_offset })
}
};
names.push(&link.name);
entries.push(SymbolTableEntry {
name_offset,
obj_header_addr,
cache,
});
}
let mut ordered: Vec<(&str, SymbolTableEntry)> = names.into_iter().zip(entries).collect();
ordered.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes()));
if let Some(w) = ordered.windows(2).find(|w| w[0].0 == w[1].0) {
return Err(IoError::InvalidState(format!(
"a symbol-table group cannot hold two links named '{}'",
w[0].0
)));
}
let heap_bytes = heap.as_bytes().to_vec();
let heap_hdr_size = local_heap_header_size(sa, ss) as u64;
let heap_addr = allocator.allocate(heap_hdr_size, FreeSpaceClass::Metadata);
let heap_data_addr = allocator.allocate(heap_bytes.len() as u64, FreeSpaceClass::Metadata);
let heap_hdr = LocalHeapHeader {
data_size: heap_bytes.len() as u64,
free_list_offset: LOCAL_HEAP_FREE_NULL,
data_addr: heap_data_addr,
};
let snod_size = cfg.symbol_table_node_size(sa, ss);
let node_size = cfg.snode_btree_node_size(sa, ss);
let leaf_capacity = cfg.sym_leaf_max_entries() as usize;
let node_capacity = cfg.snode_max_entries() as usize;
let mut pending: Vec<(u64, Vec<u8>)> = Vec::new();
let mut level: Vec<(u64, u64)> = Vec::new(); for chunk in ordered.chunks(leaf_capacity.max(1)) {
let node = SymbolTableNode {
entries: chunk.iter().map(|(_, e)| e.clone()).collect(),
};
let addr = allocator.allocate(snod_size as u64, FreeSpaceClass::Metadata);
pending.push((addr, node.encode(snod_size, sa, ss)?));
level.push((addr, chunk[chunk.len() - 1].1.name_offset));
}
let mut tree_level: u8 = 0;
let root_addr = loop {
let groups: Vec<Vec<(u64, u64)>> = if level.is_empty() {
vec![Vec::new()]
} else {
level
.chunks(node_capacity.max(1))
.map(<[(u64, u64)]>::to_vec)
.collect()
};
let addrs: Vec<u64> = groups
.iter()
.map(|_| allocator.allocate(node_size as u64, FreeSpaceClass::Metadata))
.collect();
let mut next: Vec<(u64, u64)> = Vec::with_capacity(groups.len());
for (i, group) in groups.iter().enumerate() {
let left_bound = if i == 0 {
0
} else {
groups[i - 1].last().map_or(0, |&(_, r)| r)
};
let mut keys = Vec::with_capacity(group.len() + 1);
keys.push(left_bound);
keys.extend(group.iter().map(|&(_, right)| right));
let node = BTreeV1Node {
node_type: 0,
level: tree_level,
entries_used: group.len() as u16,
left_sibling: if i == 0 { UNDEF_ADDR } else { addrs[i - 1] },
right_sibling: if i + 1 == groups.len() {
UNDEF_ADDR
} else {
addrs[i + 1]
},
keys,
children: group.iter().map(|&(a, _)| a).collect(),
};
pending.push((addrs[i], node.encode(node_size, sa, ss)?));
next.push((addrs[i], group.last().map_or(0, |&(_, right)| right)));
}
if next.len() == 1 {
break addrs[0];
}
level = next;
tree_level += 1;
};
handle.write_at(heap_addr, &heap_hdr.encode(sa, ss))?;
handle.write_at(heap_data_addr, &heap_bytes)?;
for (addr, bytes) in pending {
handle.write_at(addr, &bytes)?;
}
Ok(Stab {
btree_addr: root_addr,
heap_addr,
})
}
pub(crate) fn free_stab(allocator: &FileAllocator, extents: &StabExtents) {
for &(addr, len) in &extents.blocks {
if addr != 0 && addr != UNDEF_ADDR && len > 0 {
allocator.free(addr, len, FreeSpaceClass::Metadata);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::btree_v1::BTreeV1Config;
fn meta() -> FileMeta {
FileMeta {
ctx: FormatContext::default_v3(),
btree: BTreeV1Config::default(),
sohm: None,
}
}
struct Scratch {
dir: std::path::PathBuf,
handle: FileHandle,
allocator: FileAllocator,
}
impl Scratch {
fn new(label: &str) -> Self {
let dir = std::env::temp_dir().join(format!(
"rust_hdf5_stab_io_{}_{}_{label}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&dir).unwrap();
let handle = FileHandle::create(&dir.join("stab.bin")).unwrap();
Self {
dir,
handle,
allocator: FileAllocator::new(96),
}
}
}
impl Drop for Scratch {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.dir);
}
}
fn hard(name: &str, addr: u64) -> StabLink {
StabLink {
name: name.to_string(),
target: StabTarget::Hard { addr, cached: None },
}
}
#[test]
fn a_rebuilt_symbol_table_matches_the_shape_libhdf5_wrote() {
let s = Scratch::new("shape");
let meta = meta();
let links = vec![
hard("alpha", 0x320),
hard("beta", 0x578),
hard("gamma", 0x688),
];
let stab = write_stab(&s.handle, &s.allocator, &meta, &links).unwrap();
let node_size = meta.btree.snode_btree_node_size(8, 8);
let tree = BTreeV1Node::decode(
&s.handle.read_at(stab.btree_addr, node_size).unwrap(),
8,
8,
meta.btree.snode_max_entries(),
)
.unwrap();
assert_eq!(tree.level, 0);
assert_eq!(tree.entries_used, 1);
assert_eq!(tree.keys, vec![0, 24]);
assert_eq!(tree.left_sibling, UNDEF_ADDR);
assert_eq!(tree.right_sibling, UNDEF_ADDR);
let snod_size = meta.btree.symbol_table_node_size(8, 8);
let snod = SymbolTableNode::decode(
&s.handle.read_at(tree.children[0], snod_size).unwrap(),
8,
8,
meta.btree.sym_leaf_max_entries(),
)
.unwrap();
assert_eq!(
snod.entries
.iter()
.map(|e| e.name_offset)
.collect::<Vec<_>>(),
vec![8, 16, 24]
);
assert_eq!(
snod.entries
.iter()
.map(|e| e.obj_header_addr)
.collect::<Vec<_>>(),
vec![0x320, 0x578, 0x688]
);
let heap_hdr =
LocalHeapHeader::decode(&s.handle.read_at(stab.heap_addr, 32).unwrap(), 8, 8).unwrap();
let heap = s
.handle
.read_at(heap_hdr.data_addr, heap_hdr.data_size as usize)
.unwrap();
assert_eq!(local_heap_get_string(&heap, 0).unwrap(), "");
assert_eq!(local_heap_get_string(&heap, 8).unwrap(), "alpha");
assert_eq!(heap_hdr.free_list_offset, LOCAL_HEAP_FREE_NULL);
}
#[test]
fn a_rebuilt_symbol_table_sorts_its_entries_and_not_its_heap() {
let s = Scratch::new("sort");
let meta = meta();
let links = vec![
hard("zulu", 0x100),
hard("alpha", 0x200),
hard("mike", 0x300),
];
let stab = write_stab(&s.handle, &s.allocator, &meta, &links).unwrap();
let read = read_stab(&s.handle, &meta, stab).unwrap();
assert_eq!(
read.links
.iter()
.map(|l| l.name.as_str())
.collect::<Vec<_>>(),
vec!["alpha", "mike", "zulu"]
);
let heap_hdr =
LocalHeapHeader::decode(&s.handle.read_at(stab.heap_addr, 32).unwrap(), 8, 8).unwrap();
let heap = s
.handle
.read_at(heap_hdr.data_addr, heap_hdr.data_size as usize)
.unwrap();
assert_eq!(local_heap_get_string(&heap, 8).unwrap(), "zulu");
assert_eq!(local_heap_get_string(&heap, 16).unwrap(), "alpha");
}
#[test]
fn a_rebuilt_symbol_table_grows_leaves_then_levels() {
let meta = meta();
let leaf_capacity = u64::from(meta.btree.sym_leaf_max_entries());
let node_capacity = u64::from(meta.btree.snode_max_entries());
let count = leaf_capacity * node_capacity + 1;
let s = Scratch::new("levels");
let links: Vec<StabLink> = (0..count)
.map(|i| hard(&format!("obj{i:05}"), 0x1000 + i * 8))
.collect();
let stab = write_stab(&s.handle, &s.allocator, &meta, &links).unwrap();
let node_size = meta.btree.snode_btree_node_size(8, 8);
let root = BTreeV1Node::decode(
&s.handle.read_at(stab.btree_addr, node_size).unwrap(),
8,
8,
meta.btree.snode_max_entries(),
)
.unwrap();
assert_eq!(root.level, 1, "{count} links must not fit one tree level");
let read = read_stab(&s.handle, &meta, stab).unwrap();
assert_eq!(read.links.len(), count as usize);
let mut names: Vec<&str> = read.links.iter().map(|l| l.name.as_str()).collect();
let mut expected: Vec<&str> = links.iter().map(|l| l.name.as_str()).collect();
names.sort_unstable();
expected.sort_unstable();
assert_eq!(names, expected);
}
#[test]
fn a_rebuilt_symbol_table_links_its_siblings_both_ways() {
let meta = meta();
let count = u64::from(meta.btree.sym_leaf_max_entries())
* u64::from(meta.btree.snode_max_entries())
* 2;
let s = Scratch::new("siblings");
let links: Vec<StabLink> = (0..count)
.map(|i| hard(&format!("obj{i:05}"), 0x1000 + i * 8))
.collect();
let stab = write_stab(&s.handle, &s.allocator, &meta, &links).unwrap();
let node_size = meta.btree.snode_btree_node_size(8, 8);
let decode = |addr: u64| {
BTreeV1Node::decode(
&s.handle.read_at(addr, node_size).unwrap(),
8,
8,
meta.btree.snode_max_entries(),
)
.unwrap()
};
let root = decode(stab.btree_addr);
assert_eq!(root.level, 1);
assert!(root.children.len() >= 2);
let mut addr = root.children[0];
let mut node = decode(addr);
assert_eq!(node.left_sibling, UNDEF_ADDR);
let mut seen = 1;
while node.right_sibling != UNDEF_ADDR {
let next = decode(node.right_sibling);
assert_eq!(next.left_sibling, addr);
addr = node.right_sibling;
node = next;
seen += 1;
}
assert_eq!(seen, root.children.len());
}
#[test]
fn a_rebuilt_symbol_table_carries_a_soft_link_in_its_own_heap() {
let s = Scratch::new("soft");
let meta = meta();
let links = vec![
hard("real", 0x320),
StabLink {
name: "link".to_string(),
target: StabTarget::Soft {
value: "/real".to_string(),
},
},
];
let stab = write_stab(&s.handle, &s.allocator, &meta, &links).unwrap();
let read = read_stab(&s.handle, &meta, stab).unwrap();
assert_eq!(read.links.len(), 2);
let soft = read.links.iter().find(|l| l.name == "link").unwrap();
assert_eq!(
soft.target,
StabTarget::Soft {
value: "/real".to_string()
}
);
let snod_size = meta.btree.symbol_table_node_size(8, 8);
let node_size = meta.btree.snode_btree_node_size(8, 8);
let tree = BTreeV1Node::decode(
&s.handle.read_at(stab.btree_addr, node_size).unwrap(),
8,
8,
meta.btree.snode_max_entries(),
)
.unwrap();
let snod = SymbolTableNode::decode(
&s.handle.read_at(tree.children[0], snod_size).unwrap(),
8,
8,
meta.btree.sym_leaf_max_entries(),
)
.unwrap();
assert_eq!(snod.entries[0].obj_header_addr, UNDEF_ADDR);
assert!(matches!(
snod.entries[0].cache,
SymbolTableCache::SoftLink { .. }
));
}
#[test]
fn a_rebuilt_symbol_table_keeps_a_child_groups_cached_pair() {
let s = Scratch::new("cached");
let meta = meta();
let cached = Stab {
btree_addr: 0x348,
heap_addr: 0x568,
};
let links = vec![StabLink {
name: "sub".to_string(),
target: StabTarget::Hard {
addr: 0x320,
cached: Some(cached),
},
}];
let stab = write_stab(&s.handle, &s.allocator, &meta, &links).unwrap();
let read = read_stab(&s.handle, &meta, stab).unwrap();
assert_eq!(
read.links[0].target,
StabTarget::Hard {
addr: 0x320,
cached: Some(cached)
}
);
}
#[test]
fn an_empty_group_still_gets_a_heap_and_a_btree_root() {
let s = Scratch::new("empty");
let meta = meta();
let stab = write_stab(&s.handle, &s.allocator, &meta, &[]).unwrap();
let node_size = meta.btree.snode_btree_node_size(8, 8);
let root = BTreeV1Node::decode(
&s.handle.read_at(stab.btree_addr, node_size).unwrap(),
8,
8,
meta.btree.snode_max_entries(),
)
.unwrap();
assert_eq!(root.level, 0);
assert_eq!(root.entries_used, 0);
assert!(root.children.is_empty());
let read = read_stab(&s.handle, &meta, stab).unwrap();
assert!(read.links.is_empty());
}
#[test]
fn a_group_cannot_hold_two_links_of_one_name() {
let s = Scratch::new("dup");
let meta = meta();
let links = vec![hard("same", 0x100), hard("same", 0x200)];
let err = write_stab(&s.handle, &s.allocator, &meta, &links).unwrap_err();
assert!(
matches!(&err, IoError::InvalidState(m) if m.contains("same")),
"{err:?}"
);
}
#[test]
fn a_superseded_symbol_table_returns_its_blocks() {
let s = Scratch::new("free");
let meta = meta();
let links = [hard("a", 0x100), hard("b", 0x200)];
let first = write_stab(&s.handle, &s.allocator, &meta, &links).unwrap();
let before = s.allocator.eof();
let contents = read_stab(&s.handle, &meta, first).unwrap();
assert_eq!(contents.extents.blocks.len(), 4);
free_stab(&s.allocator, &contents.extents);
let second = write_stab(&s.handle, &s.allocator, &meta, &links).unwrap();
assert_eq!(
s.allocator.eof(),
before,
"the rewrite reused the freed blocks"
);
assert_eq!(read_stab(&s.handle, &meta, second).unwrap().links.len(), 2);
}
#[test]
fn a_symbol_table_message_round_trips_its_pair() {
let ctx = FormatContext::default_v3();
let stab = Stab {
btree_addr: 0x88,
heap_addr: 0x2a8,
};
let body = stab.encode(&ctx);
assert_eq!(body.len(), 16);
assert_eq!(&body[..8], &0x88u64.to_le_bytes());
assert_eq!(&body[8..], &0x2a8u64.to_le_bytes());
assert_eq!(Stab::decode(&body, &ctx), Some(stab));
assert_eq!(Stab::decode(&body[..15], &ctx), None);
}
}