use crate::format::checksum::checksum_metadata;
use crate::format::selection::Selection;
use crate::format::{FormatContext, FormatError, FormatResult};
const ENC_VERS_0: u8 = 0;
const ENC_VERS_1: u8 = 1;
const SOURCE_FILE_SHARED: u8 = 0x01;
const SOURCE_DSET_SHARED: u8 = 0x02;
const SOURCE_SAME_FILE: u8 = 0x04;
const ALL_FLAGS: u8 = SOURCE_FILE_SHARED | SOURCE_DSET_SHARED | SOURCE_SAME_FILE;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct VirtualMapping {
pub source_file_name: String,
pub source_dset_name: String,
pub source_selection: Selection,
pub virtual_selection: Selection,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ParsedSourceName {
segments: Vec<String>,
}
impl ParsedSourceName {
pub fn nsubs(&self) -> usize {
self.segments.len() - 1
}
pub fn build(&self, blockno: u64) -> String {
self.segments.join(&blockno.to_string())
}
}
pub fn parse_source_name(name: &str) -> FormatResult<ParsedSourceName> {
let mut segments = vec![String::new()];
let mut rest = name;
while let Some(pct) = rest.find('%') {
let (literal, tail) = rest.split_at(pct);
segments.last_mut().expect("never empty").push_str(literal);
match tail.as_bytes().get(1) {
Some(b'b') => segments.push(String::new()),
Some(b'%') => segments.last_mut().expect("never empty").push('%'),
_ => {
return Err(FormatError::InvalidData(format!(
"invalid format specifier in virtual dataset source name {name:?}: only \
%b (block index) and %% (escaped percent) are legal"
)))
}
}
rest = &tail[2.min(tail.len())..];
}
segments.last_mut().expect("never empty").push_str(rest);
Ok(ParsedSourceName { segments })
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct VirtualMappingList {
pub mappings: Vec<VirtualMapping>,
}
impl VirtualMappingList {
pub fn encode(&self, ctx: &FormatContext) -> FormatResult<Vec<u8>> {
let ss = ctx.sizeof_size as usize;
let mut buf = Vec::new();
buf.push(ENC_VERS_0);
buf.extend_from_slice(&(self.mappings.len() as u64).to_le_bytes()[..ss]);
for m in &self.mappings {
push_cstr(&mut buf, &m.source_file_name, "source file")?;
push_cstr(&mut buf, &m.source_dset_name, "source dataset")?;
buf.extend_from_slice(&m.source_selection.encode()?);
buf.extend_from_slice(&m.virtual_selection.encode()?);
}
let cksum = checksum_metadata(&buf);
buf.extend_from_slice(&cksum.to_le_bytes());
Ok(buf)
}
pub fn decode(buf: &[u8], ctx: &FormatContext) -> FormatResult<Self> {
let ss = ctx.sizeof_size as usize;
if buf.is_empty() {
return Err(FormatError::BufferTooShort {
needed: 1,
available: 0,
});
}
let heap_version = buf[0];
if heap_version != ENC_VERS_0 && heap_version != ENC_VERS_1 {
return Err(FormatError::InvalidVersion(heap_version));
}
let mut pos = 1;
if buf.len() < pos + ss {
return Err(FormatError::BufferTooShort {
needed: pos + ss,
available: buf.len(),
});
}
let num_entries = crate::format::bytes::read_le_uint(&buf[pos..], ss) as usize;
pos += ss;
let mut mappings: Vec<VirtualMapping> = Vec::new();
for i in 0..num_entries {
let flags = if heap_version >= ENC_VERS_1 {
if buf.len() < pos + 1 {
return Err(FormatError::BufferTooShort {
needed: pos + 1,
available: buf.len(),
});
}
let f = buf[pos];
pos += 1;
if f & !ALL_FLAGS != 0 {
return Err(FormatError::InvalidData(format!(
"unknown virtual dataset mapping flag bits in {f:#x}"
)));
}
f
} else {
0
};
let source_file_name = if flags & SOURCE_SAME_FILE != 0 {
".".to_string()
} else if flags & SOURCE_FILE_SHARED != 0 {
let origin = read_origin_index(buf, &mut pos, ss, i)?;
mappings[origin].source_file_name.clone()
} else {
read_cstr(buf, &mut pos)?
};
let source_dset_name = if flags & SOURCE_DSET_SHARED != 0 {
let origin = read_origin_index(buf, &mut pos, ss, i)?;
mappings[origin].source_dset_name.clone()
} else {
read_cstr(buf, &mut pos)?
};
let (source_selection, consumed) = Selection::decode(&buf[pos..])?;
pos += consumed;
let (virtual_selection, consumed) = Selection::decode(&buf[pos..])?;
pos += consumed;
parse_source_name(&source_file_name)?;
parse_source_name(&source_dset_name)?;
mappings.push(VirtualMapping {
source_file_name,
source_dset_name,
source_selection,
virtual_selection,
});
}
if buf.len() < pos + 4 {
return Err(FormatError::BufferTooShort {
needed: pos + 4,
available: buf.len(),
});
}
let stored_cksum = u32::from_le_bytes([buf[pos], buf[pos + 1], buf[pos + 2], buf[pos + 3]]);
let computed_cksum = checksum_metadata(&buf[..pos]);
if stored_cksum != computed_cksum {
return Err(FormatError::ChecksumMismatch {
expected: stored_cksum,
computed: computed_cksum,
});
}
pos += 4;
if pos != buf.len() {
return Err(FormatError::InvalidData(format!(
"virtual dataset mapping list declares {pos} bytes but the heap object holds {}",
buf.len()
)));
}
Ok(Self { mappings })
}
}
fn push_cstr(buf: &mut Vec<u8>, name: &str, what: &str) -> FormatResult<()> {
if name.as_bytes().contains(&0) {
return Err(FormatError::InvalidData(format!(
"virtual dataset {what} name {name:?} contains a NUL, which terminates a \
name on the wire"
)));
}
buf.extend_from_slice(name.as_bytes());
buf.push(0);
Ok(())
}
fn read_origin_index(
buf: &[u8],
pos: &mut usize,
ss: usize,
this_entry: usize,
) -> FormatResult<usize> {
if buf.len() < *pos + ss {
return Err(FormatError::BufferTooShort {
needed: *pos + ss,
available: buf.len(),
});
}
let origin = crate::format::bytes::read_le_uint(&buf[*pos..], ss) as usize;
*pos += ss;
if origin >= this_entry {
return Err(FormatError::InvalidData(format!(
"virtual dataset mapping entry {this_entry} shares a name with entry {origin}, \
which is not an earlier entry"
)));
}
Ok(origin)
}
fn read_cstr(buf: &[u8], pos: &mut usize) -> FormatResult<String> {
let start = *pos;
let nul = buf[start..].iter().position(|&b| b == 0).ok_or_else(|| {
FormatError::InvalidData(
"virtual dataset mapping entry has an unterminated name string".into(),
)
})?;
let s = String::from_utf8_lossy(&buf[start..start + nul]).into_owned();
*pos = start + nul + 1;
Ok(s)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::format::selection::{Hyperslab, HyperslabBlock};
fn ctx8() -> FormatContext {
FormatContext {
sizeof_addr: 8,
sizeof_size: 8,
}
}
fn all_selection_bytes() -> Vec<u8> {
let mut b = vec![0x03, 0, 0, 0]; b.extend_from_slice(&1u32.to_le_bytes()); b.extend_from_slice(&[0u8; 8]); b
}
fn single_entry_all_block() -> Vec<u8> {
let mut body = vec![ENC_VERS_0];
body.extend_from_slice(&1u64.to_le_bytes()); body.extend_from_slice(b"src.h5\0");
body.extend_from_slice(b"data\0");
body.extend_from_slice(&all_selection_bytes()); body.extend_from_slice(&all_selection_bytes()); let cksum = checksum_metadata(&body);
body.extend_from_slice(&cksum.to_le_bytes());
body
}
#[test]
fn decode_single_all_mapping() {
let buf = single_entry_all_block();
let list = VirtualMappingList::decode(&buf, &ctx8()).unwrap();
assert_eq!(list.mappings.len(), 1);
let m = &list.mappings[0];
assert_eq!(m.source_file_name, "src.h5");
assert_eq!(m.source_dset_name, "data");
assert_eq!(m.source_selection, Selection::All);
assert_eq!(m.virtual_selection, Selection::All);
}
#[test]
fn decode_empty_mapping_list() {
let mut body = vec![ENC_VERS_0];
body.extend_from_slice(&0u64.to_le_bytes());
let cksum = checksum_metadata(&body);
body.extend_from_slice(&cksum.to_le_bytes());
let list = VirtualMappingList::decode(&body, &ctx8()).unwrap();
assert!(list.mappings.is_empty());
}
#[test]
fn decode_rejects_bad_checksum() {
let mut buf = single_entry_all_block();
let last = buf.len() - 1;
buf[last] ^= 0xFF;
let err = VirtualMappingList::decode(&buf, &ctx8()).unwrap_err();
assert!(matches!(err, FormatError::ChecksumMismatch { .. }));
}
#[test]
fn decode_rejects_bad_heap_version() {
let mut buf = single_entry_all_block();
buf[0] = 2;
let err = VirtualMappingList::decode(&buf, &ctx8()).unwrap_err();
assert!(matches!(err, FormatError::InvalidVersion(2)));
}
#[test]
fn decode_rejects_unterminated_name() {
let mut body = vec![ENC_VERS_0];
body.extend_from_slice(&1u64.to_le_bytes());
body.extend_from_slice(b"no_nul_here"); let err = VirtualMappingList::decode(&body, &ctx8()).unwrap_err();
assert!(matches!(err, FormatError::InvalidData(_)));
}
#[test]
fn decode_heap_version_1_same_file() {
let mut body = vec![ENC_VERS_1];
body.extend_from_slice(&2u64.to_le_bytes());
body.push(SOURCE_SAME_FILE);
body.extend_from_slice(b"a\0");
body.extend_from_slice(&all_selection_bytes());
body.extend_from_slice(&all_selection_bytes());
body.push(SOURCE_SAME_FILE);
body.extend_from_slice(b"b\0");
body.extend_from_slice(&all_selection_bytes());
body.extend_from_slice(&all_selection_bytes());
let cksum = checksum_metadata(&body);
body.extend_from_slice(&cksum.to_le_bytes());
let list = VirtualMappingList::decode(&body, &ctx8()).unwrap();
assert_eq!(list.mappings.len(), 2);
assert_eq!(list.mappings[0].source_file_name, ".");
assert_eq!(list.mappings[1].source_file_name, ".");
assert_eq!(list.mappings[0].source_dset_name, "a");
assert_eq!(list.mappings[1].source_dset_name, "b");
}
#[test]
fn decode_heap_version_1_shared_names() {
let mut body = vec![ENC_VERS_1];
body.extend_from_slice(&2u64.to_le_bytes());
body.push(0);
body.extend_from_slice(b"shared.h5\0");
body.extend_from_slice(b"data\0");
body.extend_from_slice(&all_selection_bytes());
body.extend_from_slice(&all_selection_bytes());
body.push(SOURCE_FILE_SHARED | SOURCE_DSET_SHARED);
body.extend_from_slice(&0u64.to_le_bytes()); body.extend_from_slice(&0u64.to_le_bytes()); body.extend_from_slice(&all_selection_bytes());
body.extend_from_slice(&all_selection_bytes());
let cksum = checksum_metadata(&body);
body.extend_from_slice(&cksum.to_le_bytes());
let list = VirtualMappingList::decode(&body, &ctx8()).unwrap();
assert_eq!(list.mappings[1].source_file_name, "shared.h5");
assert_eq!(list.mappings[1].source_dset_name, "data");
}
#[test]
fn decode_rejects_non_earlier_shared_origin() {
let mut body = vec![ENC_VERS_1];
body.extend_from_slice(&1u64.to_le_bytes());
body.push(SOURCE_FILE_SHARED);
body.extend_from_slice(&0u64.to_le_bytes()); let err = VirtualMappingList::decode(&body, &ctx8()).unwrap_err();
assert!(matches!(err, FormatError::InvalidData(_)));
}
#[test]
fn decode_rejects_unknown_flag_bits() {
let mut body = vec![ENC_VERS_1];
body.extend_from_slice(&1u64.to_le_bytes());
body.push(0x08); let err = VirtualMappingList::decode(&body, &ctx8()).unwrap_err();
assert!(matches!(err, FormatError::InvalidData(_)));
}
#[test]
fn decode_mapping_with_hyperslab_virtual_selection() {
let mut hyper = vec![0x02, 0, 0, 0]; hyper.extend_from_slice(&1u32.to_le_bytes()); hyper.extend_from_slice(&[0u8; 8]);
hyper.extend_from_slice(&1u32.to_le_bytes()); hyper.extend_from_slice(&1u32.to_le_bytes()); hyper.extend_from_slice(&4u32.to_le_bytes()); hyper.extend_from_slice(&11u32.to_le_bytes());
let mut body = vec![ENC_VERS_0];
body.extend_from_slice(&1u64.to_le_bytes());
body.extend_from_slice(b"src.h5\0");
body.extend_from_slice(b"data\0");
body.extend_from_slice(&all_selection_bytes()); body.extend_from_slice(&hyper); let cksum = checksum_metadata(&body);
body.extend_from_slice(&cksum.to_le_bytes());
let list = VirtualMappingList::decode(&body, &ctx8()).unwrap();
match &list.mappings[0].virtual_selection {
Selection::Hyperslab {
rank: 1,
form: Hyperslab::Blocks(blocks),
} => {
assert_eq!(
blocks,
&vec![HyperslabBlock {
start: vec![4],
end: vec![11],
}]
);
}
other => panic!("expected a rank-1 hyperslab, got {other:?}"),
}
}
#[test]
fn decode_truncated_num_entries() {
let buf = [ENC_VERS_0, 0, 0, 0];
let err = VirtualMappingList::decode(&buf, &ctx8()).unwrap_err();
assert!(matches!(err, FormatError::BufferTooShort { .. }));
}
#[test]
fn decode_empty_buffer() {
let err = VirtualMappingList::decode(&[], &ctx8()).unwrap_err();
assert!(matches!(err, FormatError::BufferTooShort { .. }));
}
#[test]
fn encode_matches_the_captured_libhdf5_block() {
let list = VirtualMappingList {
mappings: vec![VirtualMapping {
source_file_name: "vds_src.h5".into(),
source_dset_name: "src".into(),
source_selection: Selection::All,
virtual_selection: Selection::All,
}],
};
let captured = [
0x00, 0x01, 0, 0, 0, 0, 0, 0, 0, b'v', b'd', b's', b'_', b's', b'r', b'c', b'.', b'h', b'5', 0x00, b's', b'r', b'c',
0x00, 0x03, 0, 0, 0, 0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x03, 0, 0, 0, 0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xcd, 0xe5, 0xe5, 0xed, ];
assert_eq!(list.encode(&ctx8()).unwrap(), captured);
}
#[test]
fn encode_roundtrips_a_hyperslab_mapping_at_ctx4() {
let ctx4 = FormatContext {
sizeof_addr: 4,
sizeof_size: 4,
};
let list = VirtualMappingList {
mappings: vec![
VirtualMapping {
source_file_name: "a.h5".into(),
source_dset_name: "one".into(),
source_selection: Selection::All,
virtual_selection: Selection::Hyperslab {
rank: 1,
form: Hyperslab::Blocks(vec![HyperslabBlock {
start: vec![0],
end: vec![7],
}]),
},
},
VirtualMapping {
source_file_name: "b.h5".into(),
source_dset_name: "two".into(),
source_selection: Selection::All,
virtual_selection: Selection::Hyperslab {
rank: 1,
form: Hyperslab::Blocks(vec![HyperslabBlock {
start: vec![8],
end: vec![15],
}]),
},
},
],
};
let encoded = list.encode(&ctx4).unwrap();
assert_eq!(VirtualMappingList::decode(&encoded, &ctx4).unwrap(), list);
}
#[test]
fn encode_empty_list_roundtrips() {
let list = VirtualMappingList {
mappings: Vec::new(),
};
let encoded = list.encode(&ctx8()).unwrap();
assert_eq!(encoded.len(), 1 + 8 + 4);
assert_eq!(VirtualMappingList::decode(&encoded, &ctx8()).unwrap(), list);
}
#[test]
fn encode_rejects_a_name_holding_a_nul() {
let list = VirtualMappingList {
mappings: vec![VirtualMapping {
source_file_name: "sr\0c.h5".into(),
source_dset_name: "src".into(),
source_selection: Selection::All,
virtual_selection: Selection::All,
}],
};
let err = list.encode(&ctx8()).unwrap_err();
assert!(matches!(err, FormatError::InvalidData(_)), "{err:?}");
}
#[test]
fn decode_rejects_trailing_garbage() {
let mut buf = single_entry_all_block();
buf.push(0xAB);
let err = VirtualMappingList::decode(&buf, &ctx8()).unwrap_err();
assert!(matches!(err, FormatError::InvalidData(_)));
}
#[test]
fn source_names_parse_and_build_the_way_libhdf5_does() {
for (name, nsubs, block7) in [
("plain.h5", 0, "plain.h5"),
("f%b.h5", 1, "f7.h5"),
("%b", 1, "7"),
("a%b%bc", 2, "a77c"),
("od%%d", 0, "od%d"),
("%%%b%%", 1, "%7%"),
] {
let parsed = parse_source_name(name).unwrap();
assert_eq!(parsed.nsubs(), nsubs, "{name}");
assert_eq!(parsed.build(7), block7, "{name}");
}
assert_eq!(parse_source_name("b%b_%b").unwrap().build(123), "b123_123");
for bad in ["%z", "50%", "%d.h5", "%"] {
let err = parse_source_name(bad).unwrap_err();
assert!(
matches!(&err, FormatError::InvalidData(m) if m.contains("invalid format specifier")),
"{bad}: {err:?}"
);
}
}
#[test]
fn decode_rejects_an_illegal_format_specifier_in_a_stored_name() {
let list = VirtualMappingList {
mappings: vec![VirtualMapping {
source_file_name: "src.h5".into(),
source_dset_name: "d".into(),
source_selection: Selection::All,
virtual_selection: Selection::All,
}],
};
let mut buf = list.encode(&ctx8()).unwrap();
let at = buf
.windows(6)
.position(|w| w == b"src.h5")
.expect("name is inline");
buf[at..at + 6].copy_from_slice(b"s%z.h5");
let end = buf.len() - 4;
let cksum = checksum_metadata(&buf[..end]);
buf[end..].copy_from_slice(&cksum.to_le_bytes());
let err = VirtualMappingList::decode(&buf, &ctx8()).unwrap_err();
assert!(
matches!(&err, FormatError::InvalidData(m) if m.contains("invalid format specifier")),
"{err:?}"
);
}
}