pub(crate) use crate::format::bytes::read_le_addr as read_addr;
use crate::format::bytes::read_le_uint as read_size;
use crate::format::checksum::checksum_metadata;
use crate::format::{FormatContext, FormatError, FormatResult, UNDEF_ADDR};
pub const EAHD_SIGNATURE: [u8; 4] = *b"EAHD";
pub const EAIB_SIGNATURE: [u8; 4] = *b"EAIB";
pub const EADB_SIGNATURE: [u8; 4] = *b"EADB";
pub const EASB_SIGNATURE: [u8; 4] = *b"EASB";
pub const EA_VERSION: u8 = 0;
pub const EA_CLS_CHUNK: u8 = 0;
pub const EA_CLS_FILT_CHUNK: u8 = 1;
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct FilteredChunkEntry {
pub addr: u64,
pub nbytes: u64,
pub filter_mask: u32,
}
impl FilteredChunkEntry {
pub fn undef() -> Self {
Self {
addr: UNDEF_ADDR,
nbytes: 0,
filter_mask: 0,
}
}
pub fn is_undef(&self) -> bool {
self.addr == UNDEF_ADDR
}
pub fn raw_size(sizeof_addr: u8, chunk_size_len: u8) -> u8 {
sizeof_addr + chunk_size_len + 4
}
pub fn encode(&self, sizeof_addr: usize, chunk_size_len: usize) -> Vec<u8> {
let mut buf = Vec::with_capacity(sizeof_addr + chunk_size_len + 4);
buf.extend_from_slice(&self.addr.to_le_bytes()[..sizeof_addr]);
buf.extend_from_slice(&self.nbytes.to_le_bytes()[..chunk_size_len]);
buf.extend_from_slice(&self.filter_mask.to_le_bytes());
buf
}
pub fn decode(buf: &[u8], sizeof_addr: usize, chunk_size_len: usize) -> Self {
let addr = read_addr(buf, sizeof_addr);
let nbytes = read_size(&buf[sizeof_addr..], chunk_size_len);
let off = sizeof_addr + chunk_size_len;
let filter_mask = u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]]);
Self {
addr,
nbytes,
filter_mask,
}
}
}
pub fn compute_chunk_size_len(uncompressed_chunk_bytes: u64) -> u8 {
if uncompressed_chunk_bytes == 0 {
return 1;
}
let log2 = 63 - uncompressed_chunk_bytes.leading_zeros();
let len = 1 + (log2 + 8) / 8;
std::cmp::min(len, 8) as u8
}
#[derive(Debug, Clone, PartialEq)]
pub struct ExtensibleArrayHeader {
pub class_id: u8,
pub raw_elmt_size: u8,
pub max_nelmts_bits: u8,
pub idx_blk_elmts: u8,
pub data_blk_min_elmts: u8,
pub sup_blk_min_data_ptrs: u8,
pub max_dblk_page_nelmts_bits: u8,
pub num_sblks_created: u64,
pub size_sblks_created: u64,
pub num_dblks_created: u64,
pub size_dblks_created: u64,
pub max_idx_set: u64,
pub num_elmts_realized: u64,
pub idx_blk_addr: u64,
}
impl ExtensibleArrayHeader {
pub fn new_for_chunks(ctx: &FormatContext) -> Self {
Self {
class_id: EA_CLS_CHUNK,
raw_elmt_size: ctx.sizeof_addr,
max_nelmts_bits: 32,
idx_blk_elmts: 4,
data_blk_min_elmts: 16,
sup_blk_min_data_ptrs: 4,
max_dblk_page_nelmts_bits: 10,
num_sblks_created: 0,
size_sblks_created: 0,
num_dblks_created: 0,
size_dblks_created: 0,
max_idx_set: 0,
num_elmts_realized: 0,
idx_blk_addr: UNDEF_ADDR,
}
}
pub fn new_for_filtered_chunks(ctx: &FormatContext, chunk_size_len: u8) -> Self {
Self {
class_id: EA_CLS_FILT_CHUNK,
raw_elmt_size: FilteredChunkEntry::raw_size(ctx.sizeof_addr, chunk_size_len),
max_nelmts_bits: 32,
idx_blk_elmts: 4,
data_blk_min_elmts: 16,
sup_blk_min_data_ptrs: 4,
max_dblk_page_nelmts_bits: 10,
num_sblks_created: 0,
size_sblks_created: 0,
num_dblks_created: 0,
size_dblks_created: 0,
max_idx_set: 0,
num_elmts_realized: 0,
idx_blk_addr: UNDEF_ADDR,
}
}
pub fn encoded_size(&self, ctx: &FormatContext) -> usize {
let ss = ctx.sizeof_size as usize;
let sa = ctx.sizeof_addr as usize;
4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * ss + sa + 4
}
pub fn encode(&self, ctx: &FormatContext) -> Vec<u8> {
let ss = ctx.sizeof_size as usize;
let sa = ctx.sizeof_addr as usize;
let size = self.encoded_size(ctx);
let mut buf = Vec::with_capacity(size);
buf.extend_from_slice(&EAHD_SIGNATURE);
buf.push(EA_VERSION);
buf.push(self.class_id);
buf.push(self.raw_elmt_size);
buf.push(self.max_nelmts_bits);
buf.push(self.idx_blk_elmts);
buf.push(self.data_blk_min_elmts);
buf.push(self.sup_blk_min_data_ptrs);
buf.push(self.max_dblk_page_nelmts_bits);
buf.extend_from_slice(&self.num_sblks_created.to_le_bytes()[..ss]);
buf.extend_from_slice(&self.size_sblks_created.to_le_bytes()[..ss]);
buf.extend_from_slice(&self.num_dblks_created.to_le_bytes()[..ss]);
buf.extend_from_slice(&self.size_dblks_created.to_le_bytes()[..ss]);
buf.extend_from_slice(&self.max_idx_set.to_le_bytes()[..ss]);
buf.extend_from_slice(&self.num_elmts_realized.to_le_bytes()[..ss]);
buf.extend_from_slice(&self.idx_blk_addr.to_le_bytes()[..sa]);
let cksum = checksum_metadata(&buf);
buf.extend_from_slice(&cksum.to_le_bytes());
debug_assert_eq!(buf.len(), size);
buf
}
pub fn decode(buf: &[u8], ctx: &FormatContext) -> FormatResult<Self> {
let ss = ctx.sizeof_size as usize;
let sa = ctx.sizeof_addr as usize;
let min_size = 4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 6 * ss + sa + 4;
if buf.len() < min_size {
return Err(FormatError::BufferTooShort {
needed: min_size,
available: buf.len(),
});
}
if buf[0..4] != EAHD_SIGNATURE {
return Err(FormatError::InvalidSignature);
}
let version = buf[4];
if version != EA_VERSION {
return Err(FormatError::InvalidVersion(version));
}
let data_end = min_size - 4;
let stored_cksum = u32::from_le_bytes([
buf[data_end],
buf[data_end + 1],
buf[data_end + 2],
buf[data_end + 3],
]);
let computed_cksum = checksum_metadata(&buf[..data_end]);
if stored_cksum != computed_cksum {
return Err(FormatError::ChecksumMismatch {
expected: stored_cksum,
computed: computed_cksum,
});
}
let mut pos = 5;
let class_id = buf[pos];
pos += 1;
let raw_elmt_size = buf[pos];
pos += 1;
let max_nelmts_bits = buf[pos];
pos += 1;
let idx_blk_elmts = buf[pos];
pos += 1;
let data_blk_min_elmts = buf[pos];
pos += 1;
let sup_blk_min_data_ptrs = buf[pos];
pos += 1;
let max_dblk_page_nelmts_bits = buf[pos];
pos += 1;
let num_sblks_created = read_size(&buf[pos..], ss);
pos += ss;
let size_sblks_created = read_size(&buf[pos..], ss);
pos += ss;
let num_dblks_created = read_size(&buf[pos..], ss);
pos += ss;
let size_dblks_created = read_size(&buf[pos..], ss);
pos += ss;
let max_idx_set = read_size(&buf[pos..], ss);
pos += ss;
let num_elmts_realized = read_size(&buf[pos..], ss);
pos += ss;
let idx_blk_addr = read_addr(&buf[pos..], sa);
Ok(Self {
class_id,
raw_elmt_size,
max_nelmts_bits,
idx_blk_elmts,
data_blk_min_elmts,
sup_blk_min_data_ptrs,
max_dblk_page_nelmts_bits,
num_sblks_created,
size_sblks_created,
num_dblks_created,
size_dblks_created,
max_idx_set,
num_elmts_realized,
idx_blk_addr,
})
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct ExtensibleArrayIndexBlock {
pub class_id: u8,
pub header_addr: u64,
pub elements: Vec<u64>,
pub dblk_addrs: Vec<u64>,
pub sblk_addrs: Vec<u64>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct FilteredIndexBlock {
pub class_id: u8,
pub header_addr: u64,
pub elements: Vec<FilteredChunkEntry>,
pub dblk_addrs: Vec<u64>,
pub sblk_addrs: Vec<u64>,
}
impl FilteredIndexBlock {
pub fn new(
header_addr: u64,
idx_blk_elmts: u8,
ndblk_addrs: usize,
nsblk_addrs: usize,
) -> Self {
Self {
class_id: EA_CLS_FILT_CHUNK,
header_addr,
elements: vec![FilteredChunkEntry::undef(); idx_blk_elmts as usize],
dblk_addrs: vec![UNDEF_ADDR; ndblk_addrs],
sblk_addrs: vec![UNDEF_ADDR; nsblk_addrs],
}
}
pub fn encode(&self, ctx: &FormatContext, chunk_size_len: u8) -> Vec<u8> {
let sa = ctx.sizeof_addr as usize;
let elmt_size = FilteredChunkEntry::raw_size(ctx.sizeof_addr, chunk_size_len) as usize;
let size = 4
+ 1
+ 1
+ sa
+ self.elements.len() * elmt_size
+ self.dblk_addrs.len() * sa
+ self.sblk_addrs.len() * sa
+ 4;
let mut buf = Vec::with_capacity(size);
buf.extend_from_slice(&EAIB_SIGNATURE);
buf.push(EA_VERSION);
buf.push(self.class_id);
buf.extend_from_slice(&self.header_addr.to_le_bytes()[..sa]);
for elem in &self.elements {
buf.extend_from_slice(&elem.encode(sa, chunk_size_len as usize));
}
for &addr in &self.dblk_addrs {
buf.extend_from_slice(&addr.to_le_bytes()[..sa]);
}
for &addr in &self.sblk_addrs {
buf.extend_from_slice(&addr.to_le_bytes()[..sa]);
}
let cksum = checksum_metadata(&buf);
buf.extend_from_slice(&cksum.to_le_bytes());
debug_assert_eq!(buf.len(), size);
buf
}
pub fn decode(
buf: &[u8],
ctx: &FormatContext,
idx_blk_elmts: usize,
ndblk_addrs: usize,
nsblk_addrs: usize,
chunk_size_len: u8,
) -> FormatResult<Self> {
let sa = ctx.sizeof_addr as usize;
let elmt_size = FilteredChunkEntry::raw_size(ctx.sizeof_addr, chunk_size_len) as usize;
let min_size =
4 + 1 + 1 + sa + idx_blk_elmts * elmt_size + ndblk_addrs * sa + nsblk_addrs * sa + 4;
if buf.len() < min_size {
return Err(FormatError::BufferTooShort {
needed: min_size,
available: buf.len(),
});
}
if buf[0..4] != EAIB_SIGNATURE {
return Err(FormatError::InvalidSignature);
}
if buf[4] != EA_VERSION {
return Err(FormatError::InvalidVersion(buf[4]));
}
let data_end = min_size - 4;
let stored = u32::from_le_bytes([
buf[data_end],
buf[data_end + 1],
buf[data_end + 2],
buf[data_end + 3],
]);
let computed = checksum_metadata(&buf[..data_end]);
if stored != computed {
return Err(FormatError::ChecksumMismatch {
expected: stored,
computed,
});
}
let class_id = buf[5];
let mut pos = 6;
let header_addr = read_addr(&buf[pos..], sa);
pos += sa;
let mut elements = Vec::with_capacity(idx_blk_elmts);
for _ in 0..idx_blk_elmts {
elements.push(FilteredChunkEntry::decode(
&buf[pos..],
sa,
chunk_size_len as usize,
));
pos += elmt_size;
}
let mut dblk_addrs = Vec::with_capacity(ndblk_addrs);
for _ in 0..ndblk_addrs {
dblk_addrs.push(read_addr(&buf[pos..], sa));
pos += sa;
}
let mut sblk_addrs = Vec::with_capacity(nsblk_addrs);
for _ in 0..nsblk_addrs {
sblk_addrs.push(read_addr(&buf[pos..], sa));
pos += sa;
}
Ok(Self {
class_id,
header_addr,
elements,
dblk_addrs,
sblk_addrs,
})
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct FilteredDataBlock {
pub class_id: u8,
pub header_addr: u64,
pub block_offset: u64,
pub elements: Vec<FilteredChunkEntry>,
}
impl FilteredDataBlock {
pub fn new(header_addr: u64, block_offset: u64, nelmts: usize) -> Self {
Self {
class_id: EA_CLS_FILT_CHUNK,
header_addr,
block_offset,
elements: vec![FilteredChunkEntry::undef(); nelmts],
}
}
pub fn encode(&self, ctx: &FormatContext, max_nelmts_bits: u8, chunk_size_len: u8) -> Vec<u8> {
let sa = ctx.sizeof_addr as usize;
let bo_size = ExtensibleArrayDataBlock::block_offset_size(max_nelmts_bits);
let elmt_size = FilteredChunkEntry::raw_size(ctx.sizeof_addr, chunk_size_len) as usize;
let size = 4 + 1 + 1 + sa + bo_size + self.elements.len() * elmt_size + 4;
let mut buf = Vec::with_capacity(size);
buf.extend_from_slice(&EADB_SIGNATURE);
buf.push(EA_VERSION);
buf.push(self.class_id);
buf.extend_from_slice(&self.header_addr.to_le_bytes()[..sa]);
buf.extend_from_slice(&self.block_offset.to_le_bytes()[..bo_size]);
for elem in &self.elements {
buf.extend_from_slice(&elem.encode(sa, chunk_size_len as usize));
}
let cksum = checksum_metadata(&buf);
buf.extend_from_slice(&cksum.to_le_bytes());
debug_assert_eq!(buf.len(), size);
buf
}
pub fn decode(
buf: &[u8],
ctx: &FormatContext,
max_nelmts_bits: u8,
nelmts: usize,
chunk_size_len: u8,
) -> FormatResult<Self> {
let sa = ctx.sizeof_addr as usize;
let bo_size = ExtensibleArrayDataBlock::block_offset_size(max_nelmts_bits);
let elmt_size = FilteredChunkEntry::raw_size(ctx.sizeof_addr, chunk_size_len) as usize;
let min_size = nelmts
.saturating_mul(elmt_size)
.saturating_add(10 + sa + bo_size);
if buf.len() < min_size {
return Err(FormatError::BufferTooShort {
needed: min_size,
available: buf.len(),
});
}
if buf[0..4] != EADB_SIGNATURE {
return Err(FormatError::InvalidSignature);
}
if buf[4] != EA_VERSION {
return Err(FormatError::InvalidVersion(buf[4]));
}
let data_end = min_size - 4;
let stored = u32::from_le_bytes([
buf[data_end],
buf[data_end + 1],
buf[data_end + 2],
buf[data_end + 3],
]);
let computed = checksum_metadata(&buf[..data_end]);
if stored != computed {
return Err(FormatError::ChecksumMismatch {
expected: stored,
computed,
});
}
let class_id = buf[5];
let mut pos = 6;
let header_addr = read_addr(&buf[pos..], sa);
pos += sa;
let block_offset = read_size(&buf[pos..], bo_size);
pos += bo_size;
let mut elements = Vec::with_capacity(nelmts);
for _ in 0..nelmts {
elements.push(FilteredChunkEntry::decode(
&buf[pos..],
sa,
chunk_size_len as usize,
));
pos += elmt_size;
}
Ok(Self {
class_id,
header_addr,
block_offset,
elements,
})
}
}
impl ExtensibleArrayIndexBlock {
pub fn new(
header_addr: u64,
idx_blk_elmts: u8,
ndblk_addrs: usize,
nsblk_addrs: usize,
) -> Self {
Self {
class_id: EA_CLS_CHUNK,
header_addr,
elements: vec![UNDEF_ADDR; idx_blk_elmts as usize],
dblk_addrs: vec![UNDEF_ADDR; ndblk_addrs],
sblk_addrs: vec![UNDEF_ADDR; nsblk_addrs],
}
}
pub fn encoded_size(&self, ctx: &FormatContext) -> usize {
let sa = ctx.sizeof_addr as usize;
4 + 1
+ 1
+ sa
+ self.elements.len() * sa
+ self.dblk_addrs.len() * sa
+ self.sblk_addrs.len() * sa
+ 4
}
pub fn encode(&self, ctx: &FormatContext) -> Vec<u8> {
let sa = ctx.sizeof_addr as usize;
let size = self.encoded_size(ctx);
let mut buf = Vec::with_capacity(size);
buf.extend_from_slice(&EAIB_SIGNATURE);
buf.push(EA_VERSION);
buf.push(self.class_id);
buf.extend_from_slice(&self.header_addr.to_le_bytes()[..sa]);
for &elem in &self.elements {
buf.extend_from_slice(&elem.to_le_bytes()[..sa]);
}
for &addr in &self.dblk_addrs {
buf.extend_from_slice(&addr.to_le_bytes()[..sa]);
}
for &addr in &self.sblk_addrs {
buf.extend_from_slice(&addr.to_le_bytes()[..sa]);
}
let cksum = checksum_metadata(&buf);
buf.extend_from_slice(&cksum.to_le_bytes());
debug_assert_eq!(buf.len(), size);
buf
}
pub fn decode(
buf: &[u8],
ctx: &FormatContext,
idx_blk_elmts: usize,
ndblk_addrs: usize,
nsblk_addrs: usize,
) -> FormatResult<Self> {
let sa = ctx.sizeof_addr as usize;
let min_size =
4 + 1 + 1 + sa + idx_blk_elmts * sa + ndblk_addrs * sa + nsblk_addrs * sa + 4;
if buf.len() < min_size {
return Err(FormatError::BufferTooShort {
needed: min_size,
available: buf.len(),
});
}
if buf[0..4] != EAIB_SIGNATURE {
return Err(FormatError::InvalidSignature);
}
let version = buf[4];
if version != EA_VERSION {
return Err(FormatError::InvalidVersion(version));
}
let data_end = min_size - 4;
let stored_cksum = u32::from_le_bytes([
buf[data_end],
buf[data_end + 1],
buf[data_end + 2],
buf[data_end + 3],
]);
let computed_cksum = checksum_metadata(&buf[..data_end]);
if stored_cksum != computed_cksum {
return Err(FormatError::ChecksumMismatch {
expected: stored_cksum,
computed: computed_cksum,
});
}
let class_id = buf[5];
let mut pos = 6;
let header_addr = read_addr(&buf[pos..], sa);
pos += sa;
let mut elements = Vec::with_capacity(idx_blk_elmts);
for _ in 0..idx_blk_elmts {
elements.push(read_addr(&buf[pos..], sa));
pos += sa;
}
let mut dblk_addrs = Vec::with_capacity(ndblk_addrs);
for _ in 0..ndblk_addrs {
dblk_addrs.push(read_addr(&buf[pos..], sa));
pos += sa;
}
let mut sblk_addrs = Vec::with_capacity(nsblk_addrs);
for _ in 0..nsblk_addrs {
sblk_addrs.push(read_addr(&buf[pos..], sa));
pos += sa;
}
Ok(Self {
class_id,
header_addr,
elements,
dblk_addrs,
sblk_addrs,
})
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct ExtensibleArrayDataBlock {
pub class_id: u8,
pub header_addr: u64,
pub block_offset: u64,
pub elements: Vec<u64>,
}
impl ExtensibleArrayDataBlock {
pub fn new(header_addr: u64, block_offset: u64, nelmts: usize) -> Self {
Self {
class_id: EA_CLS_CHUNK,
header_addr,
block_offset,
elements: vec![UNDEF_ADDR; nelmts],
}
}
pub fn block_offset_size(max_nelmts_bits: u8) -> usize {
std::cmp::max(1, (max_nelmts_bits as usize).div_ceil(8))
}
pub fn encoded_size(&self, ctx: &FormatContext, max_nelmts_bits: u8) -> usize {
let sa = ctx.sizeof_addr as usize;
let bo_size = Self::block_offset_size(max_nelmts_bits);
4 + 1 + 1 + sa + bo_size + self.elements.len() * sa + 4
}
pub fn encode(&self, ctx: &FormatContext, max_nelmts_bits: u8) -> Vec<u8> {
let sa = ctx.sizeof_addr as usize;
let bo_size = Self::block_offset_size(max_nelmts_bits);
let size = self.encoded_size(ctx, max_nelmts_bits);
let mut buf = Vec::with_capacity(size);
buf.extend_from_slice(&EADB_SIGNATURE);
buf.push(EA_VERSION);
buf.push(self.class_id);
buf.extend_from_slice(&self.header_addr.to_le_bytes()[..sa]);
buf.extend_from_slice(&self.block_offset.to_le_bytes()[..bo_size]);
for &elem in &self.elements {
buf.extend_from_slice(&elem.to_le_bytes()[..sa]);
}
let cksum = checksum_metadata(&buf);
buf.extend_from_slice(&cksum.to_le_bytes());
debug_assert_eq!(buf.len(), size);
buf
}
pub fn decode(
buf: &[u8],
ctx: &FormatContext,
max_nelmts_bits: u8,
nelmts: usize,
) -> FormatResult<Self> {
let sa = ctx.sizeof_addr as usize;
let bo_size = Self::block_offset_size(max_nelmts_bits);
let min_size = nelmts.saturating_mul(sa).saturating_add(10 + sa + bo_size);
if buf.len() < min_size {
return Err(FormatError::BufferTooShort {
needed: min_size,
available: buf.len(),
});
}
if buf[0..4] != EADB_SIGNATURE {
return Err(FormatError::InvalidSignature);
}
let version = buf[4];
if version != EA_VERSION {
return Err(FormatError::InvalidVersion(version));
}
let data_end = min_size - 4;
let stored_cksum = u32::from_le_bytes([
buf[data_end],
buf[data_end + 1],
buf[data_end + 2],
buf[data_end + 3],
]);
let computed_cksum = checksum_metadata(&buf[..data_end]);
if stored_cksum != computed_cksum {
return Err(FormatError::ChecksumMismatch {
expected: stored_cksum,
computed: computed_cksum,
});
}
let class_id = buf[5];
let mut pos = 6;
let header_addr = read_addr(&buf[pos..], sa);
pos += sa;
let block_offset = read_size(&buf[pos..], bo_size);
pos += bo_size;
let mut elements = Vec::with_capacity(nelmts);
for _ in 0..nelmts {
elements.push(read_addr(&buf[pos..], sa));
pos += sa;
}
Ok(Self {
class_id,
header_addr,
block_offset,
elements,
})
}
}
pub fn compute_ndblk_addrs(sup_blk_min_data_ptrs: u8) -> FormatResult<usize> {
if sup_blk_min_data_ptrs == 0 {
return Err(FormatError::InvalidData(
"extensible-array sup_blk_min_data_ptrs must be non-zero".into(),
));
}
Ok(2 * (sup_blk_min_data_ptrs as usize - 1))
}
fn log2_pow2(n: u64) -> u32 {
debug_assert!(n.is_power_of_two());
n.trailing_zeros()
}
fn log2_floor(n: u64) -> u32 {
debug_assert!(n >= 1);
63 - n.leading_zeros()
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct EaSblkInfo {
pub ndblks: u64,
pub dblk_nelmts: u64,
pub start_idx: u64,
pub start_dblk: u64,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum EaLoc {
Index { elem: usize },
Dblk(EaChunkLoc),
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct EaChunkLoc {
pub sblk_idx: usize,
pub dblk_nelmts: u64,
pub offset_in_dblk: u64,
pub dblk_block_offset: u64,
pub paged: bool,
pub path: EaDblkPath,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum EaDblkPath {
Direct { idx: usize },
ViaSblk {
sblk_off: usize,
local_dblk: usize,
ndblks_in_sblk: usize,
sblk_block_offset: u64,
},
}
#[derive(Debug, Clone)]
pub struct EaGeometry {
pub idx_blk_elmts: u64,
pub data_blk_min_elmts: u64,
pub dblk_page_nelmts: u64,
pub iblock_nsblks: usize,
pub ndblk_addrs: usize,
pub nsblk_addrs: usize,
pub sblk: Vec<EaSblkInfo>,
}
impl EaGeometry {
pub fn new(
idx_blk_elmts: u8,
data_blk_min_elmts: u8,
sup_blk_min_data_ptrs: u8,
max_nelmts_bits: u8,
max_dblk_page_nelmts_bits: u8,
) -> FormatResult<Self> {
let min = data_blk_min_elmts as u64;
if min == 0 || !min.is_power_of_two() {
return Err(FormatError::InvalidData(format!(
"extensible-array data_blk_min_elmts must be a non-zero power \
of two, got {data_blk_min_elmts}"
)));
}
let sup = sup_blk_min_data_ptrs as u64;
if sup == 0 || !sup.is_power_of_two() {
return Err(FormatError::InvalidData(format!(
"extensible-array sup_blk_min_data_ptrs must be a non-zero \
power of two, got {sup_blk_min_data_ptrs}"
)));
}
let min_bits = log2_pow2(min);
if max_nelmts_bits as u32 > 64 || (max_nelmts_bits as u32) < min_bits {
return Err(FormatError::InvalidData(format!(
"extensible-array max_nelmts_bits {max_nelmts_bits} is out of \
range for data_blk_min_elmts {data_blk_min_elmts}"
)));
}
if max_dblk_page_nelmts_bits >= 64 {
return Err(FormatError::InvalidData(format!(
"extensible-array max_dblk_page_nelmts_bits \
{max_dblk_page_nelmts_bits} is too large"
)));
}
let nsblks = 1 + (max_nelmts_bits as usize - min_bits as usize);
let iblock_nsblks = 2 * log2_pow2(sup) as usize;
if iblock_nsblks > nsblks {
return Err(FormatError::InvalidData(
"extensible-array index block would hold more super blocks \
than the array contains"
.into(),
));
}
let overflow = || {
FormatError::InvalidData(
"extensible-array geometry overflows the 64-bit index space".into(),
)
};
let mut sblk = Vec::with_capacity(nsblks);
let mut start_idx = 0u64;
let mut start_dblk = 0u64;
for u in 0..nsblks {
let ndblks = 1u64 << (u / 2);
let dblk_nelmts = (1u64 << (u as u64).div_ceil(2)) * min;
sblk.push(EaSblkInfo {
ndblks,
dblk_nelmts,
start_idx,
start_dblk,
});
start_idx = start_idx
.checked_add(ndblks.checked_mul(dblk_nelmts).ok_or_else(overflow)?)
.ok_or_else(overflow)?;
start_dblk = start_dblk.checked_add(ndblks).ok_or_else(overflow)?;
}
Ok(Self {
idx_blk_elmts: idx_blk_elmts as u64,
data_blk_min_elmts: min,
dblk_page_nelmts: 1u64 << max_dblk_page_nelmts_bits,
iblock_nsblks,
ndblk_addrs: 2 * (sup_blk_min_data_ptrs as usize - 1),
nsblk_addrs: nsblks - iblock_nsblks,
sblk,
})
}
pub fn sblk_index(&self, idx: u64) -> usize {
let e = idx - self.idx_blk_elmts;
log2_floor(e / self.data_blk_min_elmts + 1) as usize
}
pub fn is_sblk_paged(&self, u: usize) -> bool {
self.sblk[u].dblk_nelmts > self.dblk_page_nelmts
}
pub fn npages(&self, u: usize) -> u64 {
if self.is_sblk_paged(u) {
self.sblk[u].dblk_nelmts / self.dblk_page_nelmts
} else {
0
}
}
pub fn dblk_page_init_size(&self, u: usize) -> usize {
(self.npages(u) as usize).div_ceil(8)
}
pub fn dblk_page_size(&self, raw_elmt_size: usize) -> usize {
self.dblk_page_nelmts as usize * raw_elmt_size + 4
}
pub fn dblk_prefix_size(&self, sizeof_addr: u8, max_nelmts_bits: u8) -> usize {
let arr_off = ExtensibleArrayDataBlock::block_offset_size(max_nelmts_bits);
4 + 1 + 1 + sizeof_addr as usize + arr_off + 4
}
pub fn locate(&self, idx: u64) -> FormatResult<EaLoc> {
if idx < self.idx_blk_elmts {
return Ok(EaLoc::Index { elem: idx as usize });
}
let sblk_idx = self.sblk_index(idx);
let Some(&s) = self.sblk.get(sblk_idx) else {
return Err(FormatError::InvalidData(format!(
"chunk index {idx} exceeds the extensible array's capacity"
)));
};
let elmt = (idx - self.idx_blk_elmts) - s.start_idx;
let local_dblk = elmt / s.dblk_nelmts;
let offset_in_dblk = elmt % s.dblk_nelmts;
let paged = s.dblk_nelmts > self.dblk_page_nelmts;
let path = if sblk_idx < self.iblock_nsblks {
let global_dblk = s.start_dblk + local_dblk;
EaDblkPath::Direct {
idx: global_dblk as usize,
}
} else {
EaDblkPath::ViaSblk {
sblk_off: sblk_idx - self.iblock_nsblks,
local_dblk: local_dblk as usize,
ndblks_in_sblk: s.ndblks as usize,
sblk_block_offset: s.start_idx,
}
};
let dblk_block_offset = match path {
EaDblkPath::Direct { idx } => s.start_idx + (idx as u64) * s.dblk_nelmts,
EaDblkPath::ViaSblk { .. } => s.start_idx + local_dblk * s.dblk_nelmts,
};
Ok(EaLoc::Dblk(EaChunkLoc {
sblk_idx,
dblk_nelmts: s.dblk_nelmts,
offset_in_dblk,
dblk_block_offset,
paged,
path,
}))
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct ExtensibleArraySuperBlock {
pub class_id: u8,
pub header_addr: u64,
pub block_offset: u64,
pub dblk_addrs: Vec<u64>,
pub page_init: Vec<u8>,
}
impl ExtensibleArraySuperBlock {
pub fn new(class_id: u8, header_addr: u64, block_offset: u64, ndblks: usize) -> Self {
Self {
class_id,
header_addr,
block_offset,
dblk_addrs: vec![UNDEF_ADDR; ndblks],
page_init: Vec::new(),
}
}
pub fn encode(&self, ctx: &FormatContext, max_nelmts_bits: u8) -> Vec<u8> {
let sa = ctx.sizeof_addr as usize;
let bo = ExtensibleArrayDataBlock::block_offset_size(max_nelmts_bits);
let size = 4 + 1 + 1 + sa + bo + self.page_init.len() + self.dblk_addrs.len() * sa + 4;
let mut buf = Vec::with_capacity(size);
buf.extend_from_slice(&EASB_SIGNATURE);
buf.push(EA_VERSION);
buf.push(self.class_id);
buf.extend_from_slice(&self.header_addr.to_le_bytes()[..sa]);
buf.extend_from_slice(&self.block_offset.to_le_bytes()[..bo]);
buf.extend_from_slice(&self.page_init);
for &a in &self.dblk_addrs {
buf.extend_from_slice(&a.to_le_bytes()[..sa]);
}
let cksum = checksum_metadata(&buf);
buf.extend_from_slice(&cksum.to_le_bytes());
debug_assert_eq!(buf.len(), size);
buf
}
pub fn decode(
buf: &[u8],
ctx: &FormatContext,
max_nelmts_bits: u8,
ndblks: usize,
page_init_total: usize,
) -> FormatResult<Self> {
let sa = ctx.sizeof_addr as usize;
let bo = ExtensibleArrayDataBlock::block_offset_size(max_nelmts_bits);
let min_size = ndblks
.saturating_mul(sa)
.saturating_add((10 + sa + bo).saturating_add(page_init_total));
if buf.len() < min_size {
return Err(FormatError::BufferTooShort {
needed: min_size,
available: buf.len(),
});
}
if buf[0..4] != EASB_SIGNATURE {
return Err(FormatError::InvalidSignature);
}
if buf[4] != EA_VERSION {
return Err(FormatError::InvalidVersion(buf[4]));
}
let data_end = min_size - 4;
let stored = u32::from_le_bytes([
buf[data_end],
buf[data_end + 1],
buf[data_end + 2],
buf[data_end + 3],
]);
let computed = checksum_metadata(&buf[..data_end]);
if stored != computed {
return Err(FormatError::ChecksumMismatch {
expected: stored,
computed,
});
}
let class_id = buf[5];
let mut pos = 6;
let header_addr = read_addr(&buf[pos..], sa);
pos += sa;
let block_offset = read_size(&buf[pos..], bo);
pos += bo;
let page_init = buf[pos..pos + page_init_total].to_vec();
pos += page_init_total;
let mut dblk_addrs = Vec::with_capacity(ndblks);
for _ in 0..ndblks {
dblk_addrs.push(read_addr(&buf[pos..], sa));
pos += sa;
}
Ok(Self {
class_id,
header_addr,
block_offset,
dblk_addrs,
page_init,
})
}
}
pub fn compute_nsblk_addrs(
idx_blk_elmts: u8,
data_blk_min_elmts: u8,
sup_blk_min_data_ptrs: u8,
max_nelmts_bits: u8,
) -> FormatResult<usize> {
Ok(EaGeometry::new(
idx_blk_elmts,
data_blk_min_elmts,
sup_blk_min_data_ptrs,
max_nelmts_bits,
10,
)?
.nsblk_addrs)
}
#[cfg(test)]
mod tests {
use super::*;
fn ctx8() -> FormatContext {
FormatContext {
sizeof_addr: 8,
sizeof_size: 8,
}
}
fn ctx4() -> FormatContext {
FormatContext {
sizeof_addr: 4,
sizeof_size: 4,
}
}
#[test]
fn header_roundtrip() {
let mut hdr = ExtensibleArrayHeader::new_for_chunks(&ctx8());
hdr.idx_blk_addr = 0x1000;
hdr.max_idx_set = 3;
hdr.num_elmts_realized = 4;
let encoded = hdr.encode(&ctx8());
assert_eq!(encoded.len(), hdr.encoded_size(&ctx8()));
assert_eq!(&encoded[..4], b"EAHD");
let decoded = ExtensibleArrayHeader::decode(&encoded, &ctx8()).unwrap();
assert_eq!(decoded, hdr);
}
#[test]
fn header_roundtrip_ctx4() {
let mut hdr = ExtensibleArrayHeader::new_for_chunks(&ctx4());
hdr.raw_elmt_size = 4;
hdr.idx_blk_addr = 0x800;
let encoded = hdr.encode(&ctx4());
let decoded = ExtensibleArrayHeader::decode(&encoded, &ctx4()).unwrap();
assert_eq!(decoded, hdr);
}
#[test]
fn header_bad_signature() {
let mut hdr = ExtensibleArrayHeader::new_for_chunks(&ctx8());
hdr.idx_blk_addr = 0x1000;
let mut encoded = hdr.encode(&ctx8());
encoded[0] = b'X';
let err = ExtensibleArrayHeader::decode(&encoded, &ctx8()).unwrap_err();
assert!(matches!(err, FormatError::InvalidSignature));
}
#[test]
fn header_checksum_mismatch() {
let mut hdr = ExtensibleArrayHeader::new_for_chunks(&ctx8());
hdr.idx_blk_addr = 0x1000;
let mut encoded = hdr.encode(&ctx8());
encoded[6] ^= 0xFF; let err = ExtensibleArrayHeader::decode(&encoded, &ctx8()).unwrap_err();
assert!(matches!(err, FormatError::ChecksumMismatch { .. }));
}
#[test]
fn index_block_roundtrip() {
let ndblk = compute_ndblk_addrs(4).unwrap();
assert_eq!(ndblk, 6);
let mut iblk = ExtensibleArrayIndexBlock::new(0x500, 4, ndblk, 0);
iblk.elements[0] = 0x1000;
iblk.elements[1] = 0x2000;
iblk.dblk_addrs[0] = 0x3000;
let encoded = iblk.encode(&ctx8());
assert_eq!(encoded.len(), iblk.encoded_size(&ctx8()));
assert_eq!(&encoded[..4], b"EAIB");
let decoded = ExtensibleArrayIndexBlock::decode(&encoded, &ctx8(), 4, ndblk, 0).unwrap();
assert_eq!(decoded, iblk);
}
#[test]
fn index_block_roundtrip_ctx4() {
let iblk = ExtensibleArrayIndexBlock::new(0x300, 4, 6, 0);
let encoded = iblk.encode(&ctx4());
let decoded = ExtensibleArrayIndexBlock::decode(&encoded, &ctx4(), 4, 6, 0).unwrap();
assert_eq!(decoded, iblk);
}
#[test]
fn index_block_bad_checksum() {
let iblk = ExtensibleArrayIndexBlock::new(0x500, 4, 6, 0);
let mut encoded = iblk.encode(&ctx8());
encoded[8] ^= 0xFF;
let err = ExtensibleArrayIndexBlock::decode(&encoded, &ctx8(), 4, 6, 0).unwrap_err();
assert!(matches!(err, FormatError::ChecksumMismatch { .. }));
}
#[test]
fn data_block_roundtrip() {
let mut dblk = ExtensibleArrayDataBlock::new(0x500, 4, 16);
dblk.elements[0] = 0xA000;
dblk.elements[5] = 0xB000;
let encoded = dblk.encode(&ctx8(), 32);
assert_eq!(encoded.len(), dblk.encoded_size(&ctx8(), 32));
assert_eq!(&encoded[..4], b"EADB");
let decoded = ExtensibleArrayDataBlock::decode(&encoded, &ctx8(), 32, 16).unwrap();
assert_eq!(decoded, dblk);
}
#[test]
fn data_block_offset_size() {
assert_eq!(ExtensibleArrayDataBlock::block_offset_size(8), 1);
assert_eq!(ExtensibleArrayDataBlock::block_offset_size(16), 2);
assert_eq!(ExtensibleArrayDataBlock::block_offset_size(32), 4);
assert_eq!(ExtensibleArrayDataBlock::block_offset_size(0), 1);
}
#[test]
fn compute_ndblk_addrs_default() {
assert_eq!(compute_ndblk_addrs(4).unwrap(), 6);
assert_eq!(compute_ndblk_addrs(2).unwrap(), 2);
assert!(compute_ndblk_addrs(0).is_err());
}
#[test]
fn compute_nsblk_addrs_default() {
assert_eq!(compute_nsblk_addrs(4, 16, 4, 32).unwrap(), 25);
}
#[test]
fn ea_geometry_rejects_malformed_params() {
assert!(EaGeometry::new(4, 17, 4, 32, 10).is_err());
assert!(EaGeometry::new(4, 0, 4, 32, 10).is_err());
assert!(EaGeometry::new(4, 16, 0, 32, 10).is_err());
assert!(EaGeometry::new(4, 16, 3, 32, 10).is_err());
assert!(EaGeometry::new(4, 16, 4, 3, 10).is_err());
assert!(EaGeometry::new(4, 16, 4, 32, 10).is_ok());
}
#[test]
fn ea_locate_rejects_out_of_capacity_index() {
let g = EaGeometry::new(4, 16, 4, 32, 10).unwrap();
assert!(g.locate(u64::MAX).is_err());
}
#[test]
fn ea_geometry_matches_libhdf5() {
let g = EaGeometry::new(4, 16, 4, 32, 10).unwrap();
assert_eq!(g.sblk.len(), 29, "nsblks");
assert_eq!(g.iblock_nsblks, 4);
assert_eq!(g.ndblk_addrs, 6);
assert_eq!(g.nsblk_addrs, 25);
let expect = [
(1u64, 16u64, 0u64, 0u64),
(1, 32, 16, 1),
(2, 32, 48, 2),
(2, 64, 112, 4),
(4, 64, 240, 6),
];
for (u, &(nd, dn, si, sd)) in expect.iter().enumerate() {
let s = g.sblk[u];
assert_eq!(
(s.ndblks, s.dblk_nelmts, s.start_idx, s.start_dblk),
(nd, dn, si, sd),
"super block {}",
u
);
}
match g.locate(4).unwrap() {
EaLoc::Dblk(l) => {
assert_eq!(l.sblk_idx, 0);
assert!(matches!(l.path, EaDblkPath::Direct { idx: 0 }));
assert_eq!(l.dblk_block_offset, 0);
}
_ => panic!("expected data block"),
}
match g.locate(20).unwrap() {
EaLoc::Dblk(l) => {
assert!(matches!(l.path, EaDblkPath::Direct { idx: 1 }));
assert_eq!(l.dblk_block_offset, 48);
}
_ => panic!("expected data block"),
}
match g.locate(244).unwrap() {
EaLoc::Dblk(l) => {
assert_eq!(l.sblk_idx, 4);
match l.path {
EaDblkPath::ViaSblk {
sblk_off,
local_dblk,
sblk_block_offset,
..
} => assert_eq!((sblk_off, local_dblk, sblk_block_offset), (0, 0, 240)),
_ => panic!("expected super-block path"),
}
}
_ => panic!("expected data block"),
}
}
}