use crate::format::bytes::{read_le_addr as read_addr, read_le_uint as read_uint};
use crate::format::{FormatError, FormatResult};
pub const BTREE_V1_SIGNATURE: [u8; 4] = *b"TREE";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct BTreeV1Config {
pub sym_leaf_k: u16,
pub snode_internal_k: u16,
pub chunk_internal_k: u16,
}
impl Default for BTreeV1Config {
fn default() -> Self {
Self {
sym_leaf_k: 4,
snode_internal_k: 16,
chunk_internal_k: 32,
}
}
}
use crate::format::superblock::symbol_table_entry_size;
impl BTreeV1Config {
pub fn sym_leaf_max_entries(&self) -> u16 {
self.sym_leaf_k.saturating_mul(2)
}
pub fn snode_max_entries(&self) -> u16 {
self.snode_internal_k.saturating_mul(2)
}
pub fn chunk_max_entries(&self) -> u16 {
self.chunk_internal_k.saturating_mul(2)
}
pub fn snode_btree_node_size(&self, sizeof_addr: usize, sizeof_size: usize) -> usize {
btree_node_size(self.snode_max_entries(), sizeof_addr, sizeof_size)
}
pub fn chunk_btree_node_size(&self, sizeof_addr: usize, rank: usize) -> usize {
btree_node_size(
self.chunk_max_entries(),
sizeof_addr,
4 + 4 + (rank + 1) * 8,
)
}
pub fn symbol_table_node_size(&self, sizeof_addr: usize, sizeof_size: usize) -> usize {
8 + (self.sym_leaf_k as usize) * 2 * symbol_table_entry_size(sizeof_addr, sizeof_size)
}
}
fn btree_node_size(two_k: u16, sizeof_addr: usize, key_size: usize) -> usize {
let two_k = two_k as usize;
8 + 2 * sizeof_addr + two_k * sizeof_addr + (two_k + 1) * key_size
}
#[derive(Debug, Clone)]
pub struct BTreeV1Node {
pub node_type: u8,
pub level: u8,
pub entries_used: u16,
pub left_sibling: u64,
pub right_sibling: u64,
pub keys: Vec<u64>,
pub children: Vec<u64>,
}
impl BTreeV1Node {
pub fn encode(
&self,
node_size: usize,
sizeof_addr: usize,
sizeof_size: usize,
) -> FormatResult<Vec<u8>> {
if self.node_type != 0 {
return Err(FormatError::UnsupportedFeature(format!(
"B-tree v1 type {} is not encoded by BTreeV1Node (only type 0, \
symbol-table nodes)",
self.node_type
)));
}
if self.keys.len() != self.children.len() + 1 {
return Err(FormatError::InvalidData(format!(
"B-tree v1 node has {} keys for {} children; a v1 node stores both \
bounds of every child, so it needs exactly one more key than children",
self.keys.len(),
self.children.len()
)));
}
if self.entries_used as usize != self.children.len() {
return Err(FormatError::InvalidData(format!(
"B-tree v1 node declares {} entries but carries {} children",
self.entries_used,
self.children.len()
)));
}
let needed =
8 + 2 * sizeof_addr + self.children.len() * (sizeof_size + sizeof_addr) + sizeof_size;
if needed > node_size {
return Err(FormatError::InvalidData(format!(
"B-tree v1 node needs {needed} bytes for {} children, more than the \
{node_size}-byte record the file's 'K' value allows",
self.children.len()
)));
}
let mut buf = Vec::with_capacity(node_size);
buf.extend_from_slice(&BTREE_V1_SIGNATURE);
buf.push(self.node_type);
buf.push(self.level);
buf.extend_from_slice(&self.entries_used.to_le_bytes());
buf.extend_from_slice(&self.left_sibling.to_le_bytes()[..sizeof_addr]);
buf.extend_from_slice(&self.right_sibling.to_le_bytes()[..sizeof_addr]);
for (i, &child) in self.children.iter().enumerate() {
buf.extend_from_slice(&self.keys[i].to_le_bytes()[..sizeof_size]);
buf.extend_from_slice(&child.to_le_bytes()[..sizeof_addr]);
}
buf.extend_from_slice(&self.keys[self.children.len()].to_le_bytes()[..sizeof_size]);
buf.resize(node_size, 0);
Ok(buf)
}
pub fn decode(
buf: &[u8],
sizeof_addr: usize,
sizeof_size: usize,
max_entries: u16,
) -> FormatResult<Self> {
let header_size = 4 + 1 + 1 + 2 + sizeof_addr * 2;
if buf.len() < header_size {
return Err(FormatError::BufferTooShort {
needed: header_size,
available: buf.len(),
});
}
if buf[0..4] != BTREE_V1_SIGNATURE {
return Err(FormatError::InvalidSignature);
}
let node_type = buf[4];
let level = buf[5];
let entries_used = u16::from_le_bytes([buf[6], buf[7]]);
if entries_used > max_entries {
return Err(FormatError::InvalidData(format!(
"B-tree v1 node declares {entries_used} entries, more than the \
{max_entries} its 'K' value allows"
)));
}
let mut pos = 8;
let left_sibling = read_addr(&buf[pos..], sizeof_addr);
pos += sizeof_addr;
let right_sibling = read_addr(&buf[pos..], sizeof_addr);
pos += sizeof_addr;
let n = entries_used as usize;
if node_type == 0 {
let key_size = sizeof_size;
let child_size = sizeof_addr;
let data_size = (n + 1) * key_size + n * child_size;
let needed = pos + data_size;
if buf.len() < needed {
return Err(FormatError::BufferTooShort {
needed,
available: buf.len(),
});
}
let mut keys = Vec::with_capacity(n + 1);
let mut children = Vec::with_capacity(n);
for _i in 0..n {
keys.push(read_uint(&buf[pos..], key_size));
pos += key_size;
children.push(read_uint(&buf[pos..], child_size));
pos += child_size;
}
keys.push(read_uint(&buf[pos..], key_size));
Ok(BTreeV1Node {
node_type,
level,
entries_used,
left_sibling,
right_sibling,
keys,
children,
})
} else {
Err(FormatError::UnsupportedFeature(format!(
"B-tree v1 type {} not supported by BTreeV1Node (use ChunkBTreeV1Node)",
node_type
)))
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ChunkKey {
pub chunk_size: u32,
pub filter_mask: u32,
pub offsets: Vec<u64>,
}
impl ChunkKey {
pub fn for_chunk(scaled: &[u64], dims: &[u64], chunk_size: u32, filter_mask: u32) -> Self {
let mut offsets: Vec<u64> = scaled
.iter()
.zip(dims)
.map(|(&s, &d)| s.saturating_mul(d))
.collect();
offsets.push(0);
Self {
chunk_size,
filter_mask,
offsets,
}
}
pub fn right_bound(scaled: &[u64], dims: &[u64]) -> Self {
let mut offsets: Vec<u64> = scaled
.iter()
.zip(dims)
.map(|(&s, &d)| s.saturating_mul(d))
.collect();
offsets.push(*dims.last().unwrap_or(&0));
Self {
chunk_size: 0,
filter_mask: 0,
offsets,
}
}
fn encoded_size(rank: usize) -> usize {
4 + 4 + (rank + 1) * 8
}
fn encode_into(&self, buf: &mut Vec<u8>) {
buf.extend_from_slice(&self.chunk_size.to_le_bytes());
buf.extend_from_slice(&self.filter_mask.to_le_bytes());
for &o in &self.offsets {
buf.extend_from_slice(&o.to_le_bytes());
}
}
}
#[derive(Debug, Clone)]
pub struct ChunkBTreeV1Node {
pub level: u8,
pub entries_used: u16,
pub left_sibling: u64,
pub right_sibling: u64,
pub keys: Vec<ChunkKey>,
pub children: Vec<u64>,
}
impl ChunkBTreeV1Node {
pub fn encode(&self, node_size: usize, sizeof_addr: usize) -> FormatResult<Vec<u8>> {
if self.keys.len() != self.children.len() + 1 {
return Err(FormatError::InvalidData(format!(
"chunk B-tree v1 node has {} keys for {} children; a v1 node stores \
both bounds of every child, so it needs exactly one more key than \
children",
self.keys.len(),
self.children.len()
)));
}
if self.entries_used as usize != self.children.len() {
return Err(FormatError::InvalidData(format!(
"chunk B-tree v1 node declares {} entries but carries {} children",
self.entries_used,
self.children.len()
)));
}
let key_size = self.keys[0].offsets.len() * 8 + 8;
if let Some(k) = self
.keys
.iter()
.find(|k| k.offsets.len() * 8 + 8 != key_size)
{
return Err(FormatError::InvalidData(format!(
"chunk B-tree v1 node mixes keys of {} and {} offsets; every key in \
one tree describes the same dataset",
self.keys[0].offsets.len(),
k.offsets.len()
)));
}
let needed =
8 + 2 * sizeof_addr + self.children.len() * sizeof_addr + self.keys.len() * key_size;
if needed > node_size {
return Err(FormatError::InvalidData(format!(
"chunk B-tree v1 node needs {needed} bytes for {} children, more than \
the {node_size}-byte record the file's 'K' value allows",
self.children.len()
)));
}
let mut buf = Vec::with_capacity(node_size);
buf.extend_from_slice(&BTREE_V1_SIGNATURE);
buf.push(1);
buf.push(self.level);
buf.extend_from_slice(&self.entries_used.to_le_bytes());
buf.extend_from_slice(&self.left_sibling.to_le_bytes()[..sizeof_addr]);
buf.extend_from_slice(&self.right_sibling.to_le_bytes()[..sizeof_addr]);
for (i, &child) in self.children.iter().enumerate() {
self.keys[i].encode_into(&mut buf);
buf.extend_from_slice(&child.to_le_bytes()[..sizeof_addr]);
}
self.keys[self.children.len()].encode_into(&mut buf);
buf.resize(node_size, 0);
Ok(buf)
}
pub fn decode(
buf: &[u8],
sizeof_addr: usize,
rank: usize,
max_entries: u16,
) -> FormatResult<Self> {
let header_size = 4 + 1 + 1 + 2 + sizeof_addr * 2;
if buf.len() < header_size {
return Err(FormatError::BufferTooShort {
needed: header_size,
available: buf.len(),
});
}
if buf[0..4] != BTREE_V1_SIGNATURE {
return Err(FormatError::InvalidSignature);
}
let node_type = buf[4];
if node_type != 1 {
return Err(FormatError::UnsupportedFeature(format!(
"expected B-tree v1 chunk node (type 1), found type {node_type}"
)));
}
let level = buf[5];
let entries_used = u16::from_le_bytes([buf[6], buf[7]]);
if entries_used > max_entries {
return Err(FormatError::InvalidData(format!(
"chunk B-tree v1 node declares {entries_used} entries, more than \
the {max_entries} its 'K' value allows"
)));
}
let mut pos = 8;
let left_sibling = read_addr(&buf[pos..], sizeof_addr);
pos += sizeof_addr;
let right_sibling = read_addr(&buf[pos..], sizeof_addr);
pos += sizeof_addr;
let n = entries_used as usize;
let key_size = ChunkKey::encoded_size(rank);
let data_size = (n + 1) * key_size + n * sizeof_addr;
let needed = pos + data_size;
if buf.len() < needed {
return Err(FormatError::BufferTooShort {
needed,
available: buf.len(),
});
}
let decode_key = |slice: &[u8]| -> ChunkKey {
let chunk_size = u32::from_le_bytes([slice[0], slice[1], slice[2], slice[3]]);
let filter_mask = u32::from_le_bytes([slice[4], slice[5], slice[6], slice[7]]);
let mut offsets = Vec::with_capacity(rank + 1);
let mut o = 8;
for _ in 0..(rank + 1) {
offsets.push(read_uint(&slice[o..], 8));
o += 8;
}
ChunkKey {
chunk_size,
filter_mask,
offsets,
}
};
let mut keys = Vec::with_capacity(n + 1);
let mut children = Vec::with_capacity(n);
for _ in 0..n {
keys.push(decode_key(&buf[pos..pos + key_size]));
pos += key_size;
children.push(read_addr(&buf[pos..], sizeof_addr));
pos += sizeof_addr;
}
keys.push(decode_key(&buf[pos..pos + key_size]));
Ok(ChunkBTreeV1Node {
level,
entries_used,
left_sibling,
right_sibling,
keys,
children,
})
}
}
pub struct ChunkBTreeV1Tree {
nodes: Vec<TreeNode>,
node_size: usize,
sizeof_addr: usize,
}
struct TreeNode {
level: u8,
keys: Vec<ChunkKey>,
children: TreeChildren,
left: Option<usize>,
right: Option<usize>,
}
enum TreeChildren {
Chunks(Vec<u64>),
Nodes(Vec<usize>),
}
impl TreeChildren {
fn len(&self) -> usize {
match self {
Self::Chunks(v) => v.len(),
Self::Nodes(v) => v.len(),
}
}
}
impl ChunkBTreeV1Tree {
pub fn build(
entries: &[(ChunkKey, u64)],
end_key: ChunkKey,
config: &BTreeV1Config,
sizeof_addr: usize,
) -> Self {
let rank = end_key.offsets.len().saturating_sub(1);
let node_size = config.chunk_btree_node_size(sizeof_addr, rank);
let cap = (config.chunk_max_entries() as usize).max(1);
let mut nodes: Vec<TreeNode> = Vec::new();
if !entries.is_empty() {
let mut level_range = spread(entries.len(), cap)
.into_iter()
.scan(0usize, |start, m| {
let range = *start..*start + m;
*start += m;
Some(range)
})
.map(|r| {
let mut keys: Vec<ChunkKey> =
entries[r.clone()].iter().map(|(k, _)| k.clone()).collect();
keys.push(match entries.get(r.end) {
Some((k, _)) => k.clone(),
None => end_key.clone(),
});
TreeNode {
level: 0,
keys,
children: TreeChildren::Chunks(
entries[r].iter().map(|&(_, a)| a).collect(),
),
left: None,
right: None,
}
})
.collect::<Vec<_>>();
let mut level: u8 = 0;
loop {
let base = nodes.len();
let count = level_range.len();
for (i, mut node) in level_range.into_iter().enumerate() {
node.left = (i > 0).then(|| base + i - 1);
node.right = (i + 1 < count).then(|| base + i + 1);
nodes.push(node);
}
if count == 1 {
break;
}
let children: Vec<usize> = (base..base + count).collect();
level += 1;
let mut start = 0usize;
level_range = spread(count, cap)
.into_iter()
.map(|m| {
let run = &children[start..start + m];
start += m;
let mut keys: Vec<ChunkKey> =
run.iter().map(|&c| nodes[c].keys[0].clone()).collect();
keys.push(match children.get(start) {
Some(&next) => nodes[next].keys[0].clone(),
None => end_key.clone(),
});
TreeNode {
level,
keys,
children: TreeChildren::Nodes(run.to_vec()),
left: None,
right: None,
}
})
.collect();
}
}
Self {
nodes,
node_size,
sizeof_addr,
}
}
pub fn node_count(&self) -> usize {
self.nodes.len()
}
pub fn node_size(&self) -> usize {
self.node_size
}
pub fn root_address(&self, addrs: &[u64]) -> u64 {
match self.nodes.len() {
0 => crate::format::UNDEF_ADDR,
n => addrs[n - 1],
}
}
pub fn encode(&self, addrs: &[u64]) -> FormatResult<Vec<Vec<u8>>> {
let sibling = |i: Option<usize>| i.map_or(crate::format::UNDEF_ADDR, |j| addrs[j]);
self.nodes
.iter()
.map(|n| {
let children = match &n.children {
TreeChildren::Chunks(v) => v.clone(),
TreeChildren::Nodes(v) => v.iter().map(|&j| addrs[j]).collect(),
};
ChunkBTreeV1Node {
level: n.level,
entries_used: n.children.len() as u16,
left_sibling: sibling(n.left),
right_sibling: sibling(n.right),
keys: n.keys.clone(),
children,
}
.encode(self.node_size, self.sizeof_addr)
})
.collect()
}
}
fn spread(n: usize, cap: usize) -> Vec<usize> {
let k = n.div_ceil(cap);
if k == 0 {
return Vec::new();
}
let (base, extra) = (n / k, n % k);
(0..k).map(|i| base + usize::from(i < extra)).collect()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::UNDEF_ADDR;
fn build_group_btree(
level: u8,
keys: &[u64],
children: &[u64],
sizeof_addr: usize,
sizeof_size: usize,
) -> Vec<u8> {
assert_eq!(keys.len(), children.len() + 1);
let entries_used = children.len() as u16;
let mut buf = Vec::new();
buf.extend_from_slice(&BTREE_V1_SIGNATURE);
buf.push(0); buf.push(level);
buf.extend_from_slice(&entries_used.to_le_bytes());
buf.extend_from_slice(&UNDEF_ADDR.to_le_bytes()[..sizeof_addr]);
buf.extend_from_slice(&UNDEF_ADDR.to_le_bytes()[..sizeof_addr]);
for i in 0..children.len() {
buf.extend_from_slice(&keys[i].to_le_bytes()[..sizeof_size]);
buf.extend_from_slice(&children[i].to_le_bytes()[..sizeof_addr]);
}
buf.extend_from_slice(&keys[children.len()].to_le_bytes()[..sizeof_size]);
buf
}
#[test]
fn decode_leaf_node() {
let buf = build_group_btree(
0, &[0, 8, 16], &[0x100, 0x200], 8,
8,
);
let node = BTreeV1Node::decode(&buf, 8, 8, 32).unwrap();
assert_eq!(node.node_type, 0);
assert_eq!(node.level, 0);
assert_eq!(node.entries_used, 2);
assert_eq!(node.keys, vec![0, 8, 16]);
assert_eq!(node.children, vec![0x100, 0x200]);
assert_eq!(node.left_sibling, UNDEF_ADDR);
assert_eq!(node.right_sibling, UNDEF_ADDR);
}
#[test]
fn decode_internal_node() {
let buf = build_group_btree(
1, &[0, 100], &[0x500], 8,
8,
);
let node = BTreeV1Node::decode(&buf, 8, 8, 32).unwrap();
assert_eq!(node.level, 1);
assert_eq!(node.entries_used, 1);
assert_eq!(node.children, vec![0x500]);
}
#[test]
fn decode_single_entry() {
let buf = build_group_btree(0, &[0, 8], &[0x100], 8, 8);
let node = BTreeV1Node::decode(&buf, 8, 8, 32).unwrap();
assert_eq!(node.entries_used, 1);
assert_eq!(node.children.len(), 1);
}
#[test]
fn decode_4byte() {
let buf = build_group_btree(0, &[0, 4], &[0x80], 4, 4);
let node = BTreeV1Node::decode(&buf, 4, 4, 32).unwrap();
assert_eq!(node.entries_used, 1);
assert_eq!(node.children, vec![0x80]);
}
#[test]
fn decode_bad_sig() {
let mut buf = build_group_btree(0, &[0, 8], &[0x100], 8, 8);
buf[0] = b'X';
assert!(matches!(
BTreeV1Node::decode(&buf, 8, 8, 32).unwrap_err(),
FormatError::InvalidSignature
));
}
#[test]
fn decode_too_short() {
assert!(matches!(
BTreeV1Node::decode(&[0u8; 4], 8, 8, 32).unwrap_err(),
FormatError::BufferTooShort { .. }
));
}
#[test]
fn decode_unsupported_type() {
let mut buf = build_group_btree(0, &[0, 8], &[0x100], 8, 8);
buf[4] = 1; assert!(matches!(
BTreeV1Node::decode(&buf, 8, 8, 32).unwrap_err(),
FormatError::UnsupportedFeature(_)
));
}
fn build_chunk_btree(
level: u8,
keys: &[ChunkKey],
children: &[u64],
sizeof_addr: usize,
) -> Vec<u8> {
assert_eq!(keys.len(), children.len() + 1);
let entries_used = children.len() as u16;
let mut buf = Vec::new();
buf.extend_from_slice(&BTREE_V1_SIGNATURE);
buf.push(1); buf.push(level);
buf.extend_from_slice(&entries_used.to_le_bytes());
buf.extend_from_slice(&UNDEF_ADDR.to_le_bytes()[..sizeof_addr]); buf.extend_from_slice(&UNDEF_ADDR.to_le_bytes()[..sizeof_addr]);
let encode_key = |buf: &mut Vec<u8>, k: &ChunkKey| {
buf.extend_from_slice(&k.chunk_size.to_le_bytes());
buf.extend_from_slice(&k.filter_mask.to_le_bytes());
for &o in &k.offsets {
buf.extend_from_slice(&o.to_le_bytes());
}
};
for i in 0..children.len() {
encode_key(&mut buf, &keys[i]);
buf.extend_from_slice(&children[i].to_le_bytes()[..sizeof_addr]);
}
encode_key(&mut buf, &keys[children.len()]);
buf
}
fn chunk_key(size: u32, mask: u32, offsets: &[u64]) -> ChunkKey {
ChunkKey {
chunk_size: size,
filter_mask: mask,
offsets: offsets.to_vec(),
}
}
#[test]
fn decode_chunk_leaf_1d() {
let keys = [
chunk_key(32, 0, &[0, 0]),
chunk_key(32, 0, &[8, 0]),
chunk_key(0, 0, &[16, 0]),
];
let buf = build_chunk_btree(0, &keys, &[0x400, 0x800], 8);
let node = ChunkBTreeV1Node::decode(&buf, 8, 1, 64).unwrap();
assert_eq!(node.level, 0);
assert_eq!(node.entries_used, 2);
assert_eq!(node.children, vec![0x400, 0x800]);
assert_eq!(node.keys.len(), 3);
assert_eq!(node.keys[0].chunk_size, 32);
assert_eq!(node.keys[1].offsets, vec![8, 0]);
}
#[test]
fn decode_chunk_internal_2d() {
let keys = [chunk_key(64, 0, &[0, 0, 0]), chunk_key(64, 0, &[4, 4, 0])];
let buf = build_chunk_btree(1, &keys, &[0x1000], 8);
let node = ChunkBTreeV1Node::decode(&buf, 8, 2, 64).unwrap();
assert_eq!(node.level, 1);
assert_eq!(node.entries_used, 1);
assert_eq!(node.children, vec![0x1000]);
assert_eq!(node.keys[0].offsets, vec![0, 0, 0]);
}
#[test]
fn decode_chunk_filtered_key() {
let keys = [chunk_key(17, 0x1, &[0, 0]), chunk_key(0, 0, &[8, 0])];
let buf = build_chunk_btree(0, &keys, &[0x200], 8);
let node = ChunkBTreeV1Node::decode(&buf, 8, 1, 64).unwrap();
assert_eq!(node.keys[0].chunk_size, 17);
assert_eq!(node.keys[0].filter_mask, 0x1);
}
#[test]
fn decode_chunk_rejects_group_node() {
let buf = build_group_btree(0, &[0, 8], &[0x100], 8, 8);
assert!(matches!(
ChunkBTreeV1Node::decode(&buf, 8, 1, 64).unwrap_err(),
FormatError::UnsupportedFeature(_)
));
}
#[test]
fn decode_chunk_too_short() {
assert!(matches!(
ChunkBTreeV1Node::decode(&[0u8; 4], 8, 1, 64).unwrap_err(),
FormatError::BufferTooShort { .. }
));
}
#[test]
fn decode_chunk_bad_sig() {
let keys = [chunk_key(8, 0, &[0, 0]), chunk_key(0, 0, &[8, 0])];
let mut buf = build_chunk_btree(0, &keys, &[0x100], 8);
buf[0] = b'X';
assert!(matches!(
ChunkBTreeV1Node::decode(&buf, 8, 1, 64).unwrap_err(),
FormatError::InvalidSignature
));
}
#[test]
fn node_sizes_match_upstream_formula() {
let cfg = BTreeV1Config::default();
assert_eq!(cfg.snode_btree_node_size(8, 8), 8 + 16 + 32 * 8 + 33 * 8);
assert_eq!(cfg.chunk_btree_node_size(8, 1), 8 + 16 + 64 * 8 + 65 * 24);
assert_eq!(cfg.symbol_table_node_size(8, 8), 8 + 8 * 40);
}
#[test]
fn non_default_k_scales_every_node_size() {
let cfg = BTreeV1Config {
sym_leaf_k: 128,
snode_internal_k: 512,
chunk_internal_k: 256,
};
assert_eq!(cfg.snode_max_entries(), 1024);
assert_eq!(cfg.chunk_max_entries(), 512);
assert!(cfg.snode_btree_node_size(8, 8) > 8192);
assert!(cfg.chunk_btree_node_size(8, 1) > 8192);
assert!(cfg.symbol_table_node_size(8, 8) > 8192);
}
#[test]
fn decode_rejects_entries_beyond_two_k() {
let buf = build_group_btree(0, &[0, 8, 16], &[0x100, 0x200], 8, 8);
assert!(matches!(
BTreeV1Node::decode(&buf, 8, 8, 0).unwrap_err(),
FormatError::InvalidData(_)
));
assert!(BTreeV1Node::decode(&buf, 8, 8, 2).is_ok());
}
#[test]
fn decode_chunk_rejects_entries_beyond_two_k() {
let keys = [
chunk_key(32, 0, &[0, 0]),
chunk_key(32, 0, &[8, 0]),
chunk_key(0, 0, &[16, 0]),
];
let buf = build_chunk_btree(0, &keys, &[0x400, 0x800], 8);
assert!(matches!(
ChunkBTreeV1Node::decode(&buf, 8, 1, 1).unwrap_err(),
FormatError::InvalidData(_)
));
assert!(ChunkBTreeV1Node::decode(&buf, 8, 1, 2).is_ok());
}
#[test]
fn decode_chunk_4byte_addr() {
let keys = [chunk_key(16, 0, &[0, 0]), chunk_key(0, 0, &[4, 0])];
let buf = build_chunk_btree(0, &keys, &[0x80], 4);
let node = ChunkBTreeV1Node::decode(&buf, 4, 1, 64).unwrap();
assert_eq!(node.children, vec![0x80]);
}
#[test]
fn an_encoded_group_btree_node_matches_the_bytes_libhdf5_wrote() {
let node = BTreeV1Node {
node_type: 0,
level: 0,
entries_used: 1,
left_sibling: UNDEF_ADDR,
right_sibling: UNDEF_ADDR,
keys: vec![0, 24],
children: vec![0x430],
};
let node_size = BTreeV1Config::default().snode_btree_node_size(8, 8);
assert_eq!(node_size, 544);
let encoded = node.encode(node_size, 8, 8).unwrap();
let mut expected = Vec::new();
expected.extend_from_slice(b"TREE");
expected.extend_from_slice(&[0, 0, 1, 0]); expected.extend_from_slice(&[0xff; 16]); expected.extend_from_slice(&0u64.to_le_bytes()); expected.extend_from_slice(&0x430u64.to_le_bytes()); expected.extend_from_slice(&24u64.to_le_bytes()); assert_eq!(&encoded[..expected.len()], &expected[..]);
assert!(encoded[expected.len()..].iter().all(|&b| b == 0));
assert_eq!(encoded.len(), node_size);
}
#[test]
fn an_encoded_group_btree_node_round_trips_an_interior_level() {
let cfg = BTreeV1Config::default();
let node_size = cfg.snode_btree_node_size(8, 8);
let node = BTreeV1Node {
node_type: 0,
level: 1,
entries_used: 2,
left_sibling: UNDEF_ADDR,
right_sibling: UNDEF_ADDR,
keys: vec![0, 896, 1600],
children: vec![0x1a2a8, 0x1a088],
};
let encoded = node.encode(node_size, 8, 8).unwrap();
let decoded = BTreeV1Node::decode(&encoded, 8, 8, cfg.snode_max_entries()).unwrap();
assert_eq!(decoded.level, 1);
assert_eq!(decoded.entries_used, 2);
assert_eq!(decoded.keys, node.keys);
assert_eq!(decoded.children, node.children);
assert_eq!(decoded.left_sibling, UNDEF_ADDR);
}
#[test]
fn an_encoded_group_btree_node_round_trips_an_empty_root() {
let cfg = BTreeV1Config::default();
let node = BTreeV1Node {
node_type: 0,
level: 0,
entries_used: 0,
left_sibling: UNDEF_ADDR,
right_sibling: UNDEF_ADDR,
keys: vec![0],
children: vec![],
};
let encoded = node.encode(cfg.snode_btree_node_size(8, 8), 8, 8).unwrap();
let decoded = BTreeV1Node::decode(&encoded, 8, 8, cfg.snode_max_entries()).unwrap();
assert_eq!(decoded.entries_used, 0);
assert!(decoded.children.is_empty());
assert_eq!(decoded.keys, vec![0]);
}
#[test]
fn a_group_btree_node_refuses_a_key_count_that_does_not_bound_its_children() {
let cfg = BTreeV1Config::default();
let node = BTreeV1Node {
node_type: 0,
level: 0,
entries_used: 2,
left_sibling: UNDEF_ADDR,
right_sibling: UNDEF_ADDR,
keys: vec![0, 8],
children: vec![0x400, 0x800],
};
assert!(matches!(
node.encode(cfg.snode_btree_node_size(8, 8), 8, 8)
.unwrap_err(),
FormatError::InvalidData(_)
));
}
fn dense_1d_tree(nchunks: u64, chunk: u64, elem: u64, cfg: &BTreeV1Config) -> ChunkBTreeV1Tree {
let dims = [chunk, elem];
let nbytes = (chunk * elem) as u32;
let entries: Vec<(ChunkKey, u64)> = (0..nchunks)
.map(|i| {
(
ChunkKey::for_chunk(&[i], &dims, nbytes, 0),
0x1000 + i * chunk * elem,
)
})
.collect();
let end = ChunkKey::right_bound(&[nchunks - 1], &dims);
ChunkBTreeV1Tree::build(&entries, end, cfg, 8)
}
#[test]
fn a_bulk_loaded_chunk_tree_keys_the_way_libhdf5_does() {
let cfg = BTreeV1Config::default();
let tree = dense_1d_tree(2, 4, 4, &cfg);
assert_eq!(tree.node_count(), 1);
assert_eq!(tree.node_size(), cfg.chunk_btree_node_size(8, 1));
let addrs = [0x578u64];
let images = tree.encode(&addrs).unwrap();
let node = ChunkBTreeV1Node::decode(&images[0], 8, 1, cfg.chunk_max_entries()).unwrap();
assert_eq!(images[0].len(), tree.node_size());
assert_eq!(node.level, 0);
assert_eq!(node.entries_used, 2);
assert_eq!(node.children, vec![0x1000, 0x1010]);
assert_eq!(node.left_sibling, UNDEF_ADDR);
assert_eq!(node.right_sibling, UNDEF_ADDR);
let offsets: Vec<&[u64]> = node.keys.iter().map(|k| k.offsets.as_slice()).collect();
assert_eq!(offsets, vec![&[0, 0], &[4, 0], &[4, 4]]);
assert_eq!(
node.keys.iter().map(|k| k.chunk_size).collect::<Vec<_>>(),
vec![16, 16, 0]
);
assert_eq!(tree.root_address(&addrs), 0x578);
}
#[test]
fn a_bulk_loaded_chunk_tree_grows_a_level_past_2k() {
let cfg = BTreeV1Config::default();
let two_k = cfg.chunk_max_entries() as u64;
let nchunks = two_k * 3 + 1;
let tree = dense_1d_tree(nchunks, 4, 4, &cfg);
assert_eq!(tree.node_count(), 5);
let addrs: Vec<u64> = (0..tree.node_count() as u64)
.map(|i| 0x1_0000 + i * 4096)
.collect();
let images = tree.encode(&addrs).unwrap();
let nodes: Vec<ChunkBTreeV1Node> = images
.iter()
.map(|img| ChunkBTreeV1Node::decode(img, 8, 1, cfg.chunk_max_entries()).unwrap())
.collect();
let (leaves, root) = nodes.split_at(4);
let root = &root[0];
assert!(leaves.iter().all(|n| n.level == 0));
assert_eq!(root.level, 1);
assert_eq!(root.entries_used, 4);
assert_eq!(root.children, addrs[..4]);
assert_eq!(root.left_sibling, UNDEF_ADDR);
assert_eq!(root.right_sibling, UNDEF_ADDR);
assert_eq!(
leaves.iter().map(|n| n.entries_used as u64).sum::<u64>(),
nchunks
);
assert!(leaves.iter().all(|n| n.entries_used as u64 <= two_k));
for (i, leaf) in leaves.iter().enumerate() {
assert_eq!(
leaf.left_sibling,
if i == 0 { UNDEF_ADDR } else { addrs[i - 1] }
);
assert_eq!(
leaf.right_sibling,
if i + 1 == leaves.len() {
UNDEF_ADDR
} else {
addrs[i + 1]
}
);
assert_eq!(root.keys[i], leaf.keys[0]);
if let Some(next) = leaves.get(i + 1) {
assert_eq!(leaf.keys[leaf.keys.len() - 1], next.keys[0]);
}
}
let end = ChunkKey::right_bound(&[nchunks - 1], &[4, 4]);
assert_eq!(*leaves[3].keys.last().unwrap(), end);
assert_eq!(*root.keys.last().unwrap(), end);
assert_eq!(end.offsets, vec![(nchunks - 1) * 4, 4]);
}
#[test]
fn an_empty_chunk_tree_has_no_root() {
let cfg = BTreeV1Config::default();
let tree = ChunkBTreeV1Tree::build(&[], ChunkKey::right_bound(&[0], &[4, 4]), &cfg, 8);
assert_eq!(tree.node_count(), 0);
assert!(tree.encode(&[]).unwrap().is_empty());
assert_eq!(tree.root_address(&[]), UNDEF_ADDR);
}
#[test]
fn a_chunk_node_wider_than_its_record_is_refused() {
let node = ChunkBTreeV1Node {
level: 0,
entries_used: 2,
left_sibling: UNDEF_ADDR,
right_sibling: UNDEF_ADDR,
keys: (0..3)
.map(|i| ChunkKey::for_chunk(&[i], &[4, 4], 16, 0))
.collect(),
children: vec![0x100, 0x200],
};
assert!(matches!(
node.encode(64, 8).unwrap_err(),
FormatError::InvalidData(_)
));
let mut broken = node.clone();
broken.keys.pop();
assert!(matches!(
broken.encode(4096, 8).unwrap_err(),
FormatError::InvalidData(_)
));
}
#[test]
fn a_chunk_btree_node_is_not_encoded_by_the_group_encoder() {
let node = BTreeV1Node {
node_type: 1,
level: 0,
entries_used: 0,
left_sibling: UNDEF_ADDR,
right_sibling: UNDEF_ADDR,
keys: vec![0],
children: vec![],
};
assert!(matches!(
node.encode(4096, 8, 8).unwrap_err(),
FormatError::UnsupportedFeature(_)
));
}
}