use std::sync::Arc;
use super::FormatFault;
use super::block::BLOCK_HEADER_LEN;
use super::filter::PartitionedFilter;
use super::key_index::{KeyRange, SegmentIndex, SeparatorIndex};
use super::page_index::PageIndex;
use crate::{DbError, DbResult};
pub(crate) const SDAT_MAGIC: [u8; 4] = *b"ASD1";
pub(crate) const SIDX_MAGIC: [u8; 4] = *b"ASX1";
pub(crate) const SIDX_FOOTER_MAGIC: [u8; 4] = *b"ASXF";
pub(crate) const FORMAT_VERSION: u16 = 1;
pub(crate) const FILE_HEADER_LEN: usize = 64;
pub(crate) const SIDX_FOOTER_LEN: usize = 24;
pub(crate) const SECTION_RECORD_LEN: usize = 32;
pub(crate) const SECTION_BLOCK_INDEX: u32 = 1;
pub(crate) const SECTION_FILTER: u32 = 2;
pub(crate) const SECTION_PROPERTIES: u32 = 3;
pub(crate) const SECTION_SEPARATOR_INDEX: u32 = 4;
pub(crate) const SECTION_KEY_RANGE: u32 = 5;
pub(crate) const SECTION_PAGE_INDEX: u32 = 0x1002;
pub(crate) const SECTION_PREFIX_FILTER: u32 = 0x1001;
pub(crate) const OPTIONAL_SECTION_MIN: u32 = 0x1000;
const INDEX_RECORD_LEN: usize = 24;
const HEADER_CRC_AT: usize = 60;
#[cfg(target_endian = "little")]
pub(crate) const NATIVE_BYTE_ORDER: u8 = 1;
#[cfg(target_endian = "big")]
pub(crate) const NATIVE_BYTE_ORDER: u8 = 2;
fn u16_at(b: &[u8], at: usize) -> u16 {
u16::from_le_bytes(b[at..at + 2].try_into().expect("2 bytes"))
}
fn u32_at(b: &[u8], at: usize) -> u32 {
u32::from_le_bytes(b[at..at + 4].try_into().expect("4 bytes"))
}
fn u64_at(b: &[u8], at: usize) -> u64 {
u64::from_le_bytes(b[at..at + 8].try_into().expect("8 bytes"))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u8)]
pub(crate) enum OrderKind {
ByHash = 0,
ByKey = 1,
}
impl OrderKind {
pub(crate) fn from_byte(b: u8) -> Option<Self> {
match b {
0 => Some(Self::ByHash),
1 => Some(Self::ByKey),
_ => None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct SegHeader {
pub segment_id: u64,
pub version: u32,
pub order: OrderKind,
pub key_len: u16,
pub seed: u64,
pub block_size: u32,
}
impl SegHeader {
pub(crate) fn encode(&self, magic: [u8; 4]) -> [u8; FILE_HEADER_LEN] {
let mut b = [0u8; FILE_HEADER_LEN];
b[0..4].copy_from_slice(&magic);
b[4..6].copy_from_slice(&FORMAT_VERSION.to_le_bytes());
b[6] = NATIVE_BYTE_ORDER;
b[7] = self.order as u8;
b[8..16].copy_from_slice(&self.segment_id.to_le_bytes());
b[16..20].copy_from_slice(&self.version.to_le_bytes());
b[20..22].copy_from_slice(&self.key_len.to_le_bytes());
b[24..32].copy_from_slice(&self.seed.to_le_bytes());
b[32..36].copy_from_slice(&self.block_size.to_le_bytes());
let crc = crc32fast::hash(&b[..HEADER_CRC_AT]);
b[HEADER_CRC_AT..].copy_from_slice(&crc.to_le_bytes());
b
}
pub(crate) fn decode(bytes: &[u8], magic: [u8; 4]) -> Result<Self, FormatFault> {
if bytes.len() < FILE_HEADER_LEN {
return Err(FormatFault::corrupt("file is shorter than its header"));
}
let b = &bytes[..FILE_HEADER_LEN];
if b[0..4] != magic {
return Err(FormatFault::corrupt(format!(
"bad magic {:?}, expected {:?}",
&b[0..4],
magic
)));
}
let stored = u32_at(b, HEADER_CRC_AT);
let computed = crc32fast::hash(&b[..HEADER_CRC_AT]);
if stored != computed {
return Err(FormatFault::corrupt(format!(
"header CRC mismatch: stored {stored:#x}, computed {computed:#x}"
)));
}
let version = u16_at(b, 4);
if version != FORMAT_VERSION {
return Err(FormatFault::unsupported(format!(
"format version {version}"
)));
}
if b[6] != NATIVE_BYTE_ORDER {
return Err(FormatFault::unsupported(format!(
"entries were written in byte order {}, this build reads {NATIVE_BYTE_ORDER}",
b[6]
)));
}
let order = match OrderKind::from_byte(b[7]) {
Some(order) => order,
None => {
return Err(FormatFault::unsupported(format!("entry order {}", b[7])));
}
};
if u16_at(b, 22) != 0 || b[36..HEADER_CRC_AT].iter().any(|&x| x != 0) {
return Err(FormatFault::unsupported("header reserved bytes are set"));
}
Ok(Self {
segment_id: u64_at(b, 8),
version: u32_at(b, 16),
order,
key_len: u16_at(b, 20),
seed: u64_at(b, 24),
block_size: u32_at(b, 32),
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct BlockIndexRecord {
pub first_h: u64,
pub offset: u64,
pub block_len: u32,
pub entries: u32,
}
pub(crate) struct BlockIndexBuilder {
records: Vec<u8>,
end: u64,
count: usize,
}
impl BlockIndexBuilder {
pub(crate) fn new() -> Self {
Self {
records: Vec::new(),
end: FILE_HEADER_LEN as u64,
count: 0,
}
}
pub(crate) fn push(&mut self, first_h: u64, block_len: u32, entries: u32) {
self.records.extend_from_slice(&first_h.to_le_bytes());
self.records.extend_from_slice(&self.end.to_le_bytes());
self.records.extend_from_slice(&block_len.to_le_bytes());
self.records.extend_from_slice(&entries.to_le_bytes());
self.end += u64::from(block_len);
self.count += 1;
}
pub(crate) fn end_offset(&self) -> u64 {
self.end
}
pub(crate) fn finish(mut self) -> Vec<u8> {
self.records.extend_from_slice(&self.end.to_le_bytes());
self.records
}
}
fn check_block_count(count: usize) -> Result<(), FormatFault> {
if u32::try_from(count).is_err() {
return Err(FormatFault::corrupt(format!(
"block index lists {count} blocks, more than u32::MAX"
)));
}
Ok(())
}
pub(crate) fn parse_index_records(
section: &[u8],
) -> Result<(Vec<BlockIndexRecord>, u64), FormatFault> {
if section.len() < 8 || !(section.len() - 8).is_multiple_of(INDEX_RECORD_LEN) {
return Err(FormatFault::corrupt(format!(
"block index section of {} bytes is not a whole number of records",
section.len()
)));
}
let count = (section.len() - 8) / INDEX_RECORD_LEN;
check_block_count(count)?;
let end = u64_at(section, count * INDEX_RECORD_LEN);
let records = parse_index_run(§ion[..count * INDEX_RECORD_LEN], 0, end)?;
Ok((records, end))
}
pub(crate) fn parse_index_run(
section: &[u8],
first_block: usize,
end: u64,
) -> Result<Vec<BlockIndexRecord>, FormatFault> {
if !section.len().is_multiple_of(INDEX_RECORD_LEN) {
return Err(FormatFault::corrupt("block index run has a partial record"));
}
let count = section.len() / INDEX_RECORD_LEN;
let mut records = Vec::with_capacity(count);
let mut expected_offset: Option<u64> = None;
let mut previous_h: Option<u64> = None;
for i in 0..count {
let at = i * INDEX_RECORD_LEN;
let rec = BlockIndexRecord {
first_h: u64_at(section, at),
offset: u64_at(section, at + 8),
block_len: u32_at(section, at + 16),
entries: u32_at(section, at + 20),
};
if rec.offset < FILE_HEADER_LEN as u64 {
return Err(FormatFault::corrupt(format!(
"block {i} starts at {} inside the file header",
rec.offset
)));
}
if first_block == 0 && i == 0 && rec.offset != FILE_HEADER_LEN as u64 {
return Err(FormatFault::corrupt(format!(
"first block starts at {}, expected {} after the file header",
rec.offset, FILE_HEADER_LEN
)));
}
if let Some(expected) = expected_offset
&& rec.offset != expected
{
return Err(FormatFault::corrupt(format!(
"block {i} starts at {}, blocks are not contiguous (expected {expected})",
rec.offset
)));
}
if previous_h.is_some_and(|h| h >= rec.first_h) {
return Err(FormatFault::corrupt(format!(
"block {i} is out of order: first hash {:#x}",
rec.first_h
)));
}
if (rec.block_len as usize) <= BLOCK_HEADER_LEN || rec.entries == 0 {
return Err(FormatFault::corrupt(format!("block {i} is empty")));
}
expected_offset = Some(
rec.offset
.checked_add(u64::from(rec.block_len))
.ok_or_else(|| FormatFault::corrupt("block index range overflows u64"))?,
);
previous_h = Some(rec.first_h);
records.push(rec);
}
let expected_end = expected_offset.unwrap_or(FILE_HEADER_LEN as u64);
if end != expected_end {
return Err(FormatFault::corrupt(format!(
"block index ends at {end}, its blocks end at {expected_end}"
)));
}
Ok(records)
}
pub(crate) struct BlockIndex {
first_h: Vec<u64>,
offsets: Vec<u64>,
}
impl BlockIndex {
pub(crate) fn parse(section: &[u8]) -> Result<Self, FormatFault> {
let (records, end) = parse_index_records(section)?;
let mut first_h = Vec::with_capacity(records.len());
let mut offsets = Vec::with_capacity(records.len() + 1);
for rec in &records {
first_h.push(rec.first_h);
offsets.push(rec.offset);
}
offsets.push(end);
Ok(Self { first_h, offsets })
}
pub(crate) fn len(&self) -> usize {
self.first_h.len()
}
#[cfg(test)]
pub(crate) fn is_empty(&self) -> bool {
self.first_h.is_empty()
}
pub(crate) fn locate(&self, h: u64) -> Option<usize> {
self.first_h
.partition_point(|&first| first <= h)
.checked_sub(1)
}
pub(crate) fn range(&self, block: usize) -> (u64, usize) {
let start = self.offsets[block];
(start, (self.offsets[block + 1] - start) as usize)
}
pub(crate) fn first_h(&self, block: usize) -> u64 {
self.first_h[block]
}
pub(crate) fn end_offset(&self) -> u64 {
*self
.offsets
.last()
.expect("the end offset is always present")
}
pub(crate) fn resident_bytes(&self) -> usize {
(self.first_h.len() + self.offsets.len()) * size_of::<u64>()
}
}
const PROPS_FIXED_LEN: usize = 11 * 8 + 2 + 2;
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct Properties {
pub entries: u64,
pub tombstones: u64,
pub min_gsn: u64,
pub max_gsn: u64,
pub min_h: u64,
pub max_h: u64,
pub raw_bytes: u64,
pub opened_at_ms: u64,
pub frozen_at_ms: u64,
pub sealed_at_ms: u64,
pub typ_hash: u64,
pub value_version: u16,
pub writer: String,
}
impl Properties {
pub(crate) fn encode(&self) -> Vec<u8> {
let writer = self.writer.as_bytes();
let writer_len = u16::try_from(writer.len()).expect("properties writer exceeds u16 length");
let mut out = Vec::with_capacity(PROPS_FIXED_LEN + writer.len());
for v in [
self.entries,
self.tombstones,
self.min_gsn,
self.max_gsn,
self.min_h,
self.max_h,
self.raw_bytes,
self.opened_at_ms,
self.frozen_at_ms,
self.sealed_at_ms,
self.typ_hash,
] {
out.extend_from_slice(&v.to_le_bytes());
}
out.extend_from_slice(&self.value_version.to_le_bytes());
out.extend_from_slice(&writer_len.to_le_bytes());
out.extend_from_slice(writer);
out
}
pub(crate) fn decode(section: &[u8]) -> Result<Self, FormatFault> {
if section.len() < PROPS_FIXED_LEN {
return Err(FormatFault::corrupt("properties section is truncated"));
}
let writer_len = u16_at(section, 90) as usize;
if section.len() != PROPS_FIXED_LEN + writer_len {
return Err(FormatFault::corrupt(format!(
"properties section is {} bytes, expected {}",
section.len(),
PROPS_FIXED_LEN + writer_len
)));
}
let writer = String::from_utf8(section[PROPS_FIXED_LEN..].to_vec())
.map_err(|_| FormatFault::corrupt("properties writer is not UTF-8"))?;
Ok(Self {
entries: u64_at(section, 0),
tombstones: u64_at(section, 8),
min_gsn: u64_at(section, 16),
max_gsn: u64_at(section, 24),
min_h: u64_at(section, 32),
max_h: u64_at(section, 40),
raw_bytes: u64_at(section, 48),
opened_at_ms: u64_at(section, 56),
frozen_at_ms: u64_at(section, 64),
sealed_at_ms: u64_at(section, 72),
typ_hash: u64_at(section, 80),
value_version: u16_at(section, 88),
writer,
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct SectionRecord {
pub kind: u32,
pub offset: u64,
pub len: u64,
pub crc32: u32,
}
pub(crate) fn encode_sidecar(header: &SegHeader, sections: &[(u32, Vec<u8>)]) -> Vec<u8> {
let bodies: usize = sections.iter().map(|(_, s)| s.len()).sum();
let mut out = Vec::with_capacity(
FILE_HEADER_LEN + bodies + sections.len() * SECTION_RECORD_LEN + SIDX_FOOTER_LEN,
);
out.extend_from_slice(&header.encode(SIDX_MAGIC));
let mut table = Vec::with_capacity(sections.len() * SECTION_RECORD_LEN);
for (kind, body) in sections {
table.extend_from_slice(&kind.to_le_bytes());
table.extend_from_slice(&0u32.to_le_bytes());
table.extend_from_slice(&(out.len() as u64).to_le_bytes());
table.extend_from_slice(&(body.len() as u64).to_le_bytes());
table.extend_from_slice(&crc32fast::hash(body).to_le_bytes());
table.extend_from_slice(&0u32.to_le_bytes());
out.extend_from_slice(body);
}
let table_offset = out.len() as u64;
out.extend_from_slice(&table);
out.extend_from_slice(&(sections.len() as u32).to_le_bytes());
out.extend_from_slice(&crc32fast::hash(&table).to_le_bytes());
out.extend_from_slice(&table_offset.to_le_bytes());
out.extend_from_slice(&SIDX_FOOTER_MAGIC);
out.extend_from_slice(&0u32.to_le_bytes());
out
}
pub(crate) fn encode_sidecar_paged(
header: &SegHeader,
sections: &[(u32, Vec<u8>)],
) -> DbResult<Vec<u8>> {
let derive = |bytes: &[u8]| -> Result<PageIndex, FormatFault> {
let footer_at = bytes.len() - SIDX_FOOTER_LEN;
let (at, count, crc) = parse_footer(&bytes[footer_at..], bytes.len() as u64)?;
let records = parse_table(&bytes[at as usize..footer_at], count, crc, at)?;
PageIndex::derive(bytes, &records, header)
};
let bytes = encode_sidecar(header, sections);
let page_index = derive(&bytes).map_err(|_| DbError::Internal("segment writer: page index"))?;
let mut sections = sections.to_vec();
sections.push((SECTION_PAGE_INDEX, page_index.encode()));
let bytes = encode_sidecar(header, §ions);
debug_assert_eq!(derive(&bytes), Ok(page_index));
Ok(bytes)
}
pub(crate) fn parse_footer(footer: &[u8], file_len: u64) -> Result<(u64, u32, u32), FormatFault> {
if footer.len() != SIDX_FOOTER_LEN {
return Err(FormatFault::corrupt("sidecar footer is truncated"));
}
if footer[16..20] != SIDX_FOOTER_MAGIC {
return Err(FormatFault::corrupt("bad sidecar footer magic"));
}
if u32_at(footer, 20) != 0 {
return Err(FormatFault::corrupt(
"sidecar footer reserved bytes are set",
));
}
let count = u32_at(footer, 0);
let table_crc = u32_at(footer, 4);
let table_offset = u64_at(footer, 8);
let expected_len = table_offset
.checked_add(u64::from(count) * SECTION_RECORD_LEN as u64)
.and_then(|n| n.checked_add(SIDX_FOOTER_LEN as u64));
if table_offset < FILE_HEADER_LEN as u64 || expected_len != Some(file_len) {
return Err(FormatFault::corrupt(format!(
"sidecar footer places {count} sections at {table_offset} in a file of {file_len} bytes"
)));
}
Ok((table_offset, count, table_crc))
}
pub(crate) fn parse_table(
table: &[u8],
count: u32,
table_crc: u32,
table_offset: u64,
) -> Result<Vec<SectionRecord>, FormatFault> {
if table.len() != count as usize * SECTION_RECORD_LEN {
return Err(FormatFault::corrupt("sidecar section table is truncated"));
}
let computed = crc32fast::hash(table);
if computed != table_crc {
return Err(FormatFault::corrupt(format!(
"sidecar section table CRC mismatch: stored {table_crc:#x}, computed {computed:#x}"
)));
}
let mut records = Vec::with_capacity(count as usize);
for i in 0..count as usize {
let rec = &table[i * SECTION_RECORD_LEN..(i + 1) * SECTION_RECORD_LEN];
if u32_at(rec, 4) != 0 || u32_at(rec, 28) != 0 {
return Err(FormatFault::unsupported(format!(
"section {i} reserved bytes are set"
)));
}
let record = SectionRecord {
kind: u32_at(rec, 0),
offset: u64_at(rec, 8),
len: u64_at(rec, 16),
crc32: u32_at(rec, 24),
};
if record.offset < FILE_HEADER_LEN as u64 {
return Err(FormatFault::corrupt(format!(
"section {i} starts at {} inside the file header",
record.offset
)));
}
match record.offset.checked_add(record.len) {
Some(end) if end <= table_offset => {}
_ => {
return Err(FormatFault::corrupt(format!(
"section {i} ({} bytes at {}) runs past the section table at {table_offset}",
record.len, record.offset
)));
}
}
records.push(record);
}
let mut by_offset: Vec<&SectionRecord> = records.iter().collect();
by_offset.sort_by_key(|r| r.offset);
for pair in by_offset.windows(2) {
if pair[0].offset + pair[0].len > pair[1].offset {
return Err(FormatFault::corrupt(format!(
"sections of kind {} and {} overlap",
pair[0].kind, pair[1].kind
)));
}
}
Ok(records)
}
pub(crate) fn check_section_table(
records: &[SectionRecord],
header: &SegHeader,
) -> Result<(), FormatFault> {
let mut seen = std::collections::HashSet::new();
for rec in records {
if rec.kind == SECTION_PREFIX_FILTER {
if header.order == OrderKind::ByKey && rec.len < 8 {
return Err(FormatFault::corrupt("invalid prefix filter header"));
}
continue;
}
if rec.kind >= OPTIONAL_SECTION_MIN && rec.kind != SECTION_PAGE_INDEX {
continue;
}
if !seen.insert(rec.kind) {
return Err(FormatFault::corrupt(format!(
"duplicate {} section",
section_name(rec.kind)
)));
}
match rec.kind {
SECTION_BLOCK_INDEX if header.order == OrderKind::ByHash => {
if rec.len < 8 || !(rec.len - 8).is_multiple_of(24) {
return Err(FormatFault::corrupt("invalid block index length"));
}
}
SECTION_SEPARATOR_INDEX | SECTION_KEY_RANGE if header.order == OrderKind::ByKey => {}
SECTION_BLOCK_INDEX | SECTION_SEPARATOR_INDEX | SECTION_KEY_RANGE => {
return Err(FormatFault::corrupt(
"sidecar sections differ from entry order",
));
}
SECTION_FILTER | SECTION_PROPERTIES | SECTION_PAGE_INDEX => {}
other => {
return Err(FormatFault::unsupported(format!(
"required section kind {other}"
)));
}
}
}
let index = if header.order == OrderKind::ByHash {
SECTION_BLOCK_INDEX
} else {
SECTION_SEPARATOR_INDEX
};
for kind in [index, SECTION_FILTER, SECTION_PROPERTIES] {
if !seen.contains(&kind) {
return Err(FormatFault::corrupt(format!(
"sidecar has no {} section",
section_name(kind)
)));
}
}
if header.order == OrderKind::ByKey && !seen.contains(&SECTION_KEY_RANGE) {
return Err(FormatFault::corrupt("sidecar has no key range section"));
}
Ok(())
}
fn section_name(kind: u32) -> &'static str {
match kind {
SECTION_BLOCK_INDEX => "block index",
SECTION_PAGE_INDEX => "page index",
SECTION_FILTER => "filter",
SECTION_PROPERTIES => "properties",
SECTION_SEPARATOR_INDEX => "separator index",
SECTION_KEY_RANGE => "key range",
SECTION_PREFIX_FILTER => "prefix filter",
_ => "unknown",
}
}
pub(crate) struct Sidecar {
pub header: SegHeader,
pub index: SegmentIndex,
pub key_range: Option<KeyRange>,
pub prefix_filters: Vec<(u16, PartitionedFilter)>,
pub filter: PartitionedFilter,
pub props: Properties,
pub page_index: Option<PageIndex>,
}
impl Sidecar {
pub(crate) fn parse(bytes: &[u8]) -> Result<Self, FormatFault> {
let header = SegHeader::decode(bytes, SIDX_MAGIC)?;
if bytes.len() < FILE_HEADER_LEN + SIDX_FOOTER_LEN {
return Err(FormatFault::corrupt("sidecar has no footer"));
}
let footer_at = bytes.len() - SIDX_FOOTER_LEN;
let (table_offset, count, table_crc) =
parse_footer(&bytes[footer_at..], bytes.len() as u64)?;
let table = &bytes[table_offset as usize..footer_at];
let records = parse_table(table, count, table_crc, table_offset)?;
check_section_table(&records, &header)?;
let mut index = None;
let mut key_range = None;
let mut prefix_filters: Vec<(u16, PartitionedFilter)> = Vec::new();
let mut filter = None;
let mut props = None;
let mut page_index_body = None;
for rec in &records {
let body = &bytes[rec.offset as usize..(rec.offset + rec.len) as usize];
let computed = crc32fast::hash(body);
if computed != rec.crc32 {
return Err(FormatFault::corrupt(format!(
"{} section CRC mismatch: stored {:#x}, computed {computed:#x}",
section_name(rec.kind),
rec.crc32
)));
}
if rec.kind == SECTION_PREFIX_FILTER && header.order == OrderKind::ByKey {
if body.len() < 8 || body[2..8].iter().any(|&b| b != 0) {
return Err(FormatFault::corrupt("invalid prefix filter header"));
}
let len = u16_at(body, 0);
if len == 0
|| len >= header.key_len
|| prefix_filters.iter().any(|(p, _)| *p == len)
{
return Err(FormatFault::corrupt("invalid or duplicate prefix length"));
}
prefix_filters.push((len, PartitionedFilter::parse(Arc::from(&body[8..]))?));
continue;
}
if rec.kind == SECTION_PAGE_INDEX {
if page_index_body.replace(body).is_some() {
return Err(FormatFault::corrupt("duplicate page index section"));
}
continue;
}
if rec.kind >= OPTIONAL_SECTION_MIN {
continue;
}
if (header.order == OrderKind::ByHash
&& matches!(rec.kind, SECTION_SEPARATOR_INDEX | SECTION_KEY_RANGE))
|| (header.order == OrderKind::ByKey && rec.kind == SECTION_BLOCK_INDEX)
{
return Err(FormatFault::corrupt(
"sidecar sections differ from entry order",
));
}
match rec.kind {
SECTION_BLOCK_INDEX if index.is_none() => {
index = Some(SegmentIndex::Hash(BlockIndex::parse(body)?))
}
SECTION_SEPARATOR_INDEX if index.is_none() => {
index = Some(SegmentIndex::Key(SeparatorIndex::parse(
body,
header.key_len as usize,
)?))
}
SECTION_KEY_RANGE if key_range.is_none() => {
key_range = Some(KeyRange::decode(body, header.key_len as usize)?)
}
SECTION_FILTER if filter.is_none() => {
filter = Some(PartitionedFilter::parse(Arc::from(body))?);
}
SECTION_PROPERTIES if props.is_none() => props = Some(Properties::decode(body)?),
SECTION_BLOCK_INDEX
| SECTION_SEPARATOR_INDEX
| SECTION_KEY_RANGE
| SECTION_FILTER
| SECTION_PROPERTIES => {
return Err(FormatFault::corrupt(format!(
"duplicate {} section",
section_name(rec.kind)
)));
}
other => {
return Err(FormatFault::unsupported(format!(
"required section kind {other}"
)));
}
}
}
let page_index = page_index_body
.map(|body| {
let Some(decoded) = PageIndex::decode_known(body, &records, &header)? else {
return Ok(None);
};
if decoded != PageIndex::derive(bytes, &records, &header)? {
return Err(FormatFault::corrupt(
"page index disagrees with the sections it describes",
));
}
Ok(Some(decoded))
})
.transpose()?
.flatten();
let missing = |name: &str| FormatFault::corrupt(format!("sidecar has no {name} section"));
if header.order == OrderKind::ByKey && key_range.is_none() {
return Err(missing("key range"));
}
Ok(Self {
header,
page_index,
key_range,
prefix_filters,
index: index.ok_or_else(|| missing("block index"))?,
filter: filter.ok_or_else(|| missing("filter"))?,
props: props.ok_or_else(|| missing("properties"))?,
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::seg::filter::PartitionedFilterBuilder;
#[test]
fn a_page_index_that_disagrees_with_its_sections_fails_the_parse() {
use crate::seg::page_index::{GroupSamples, PageIndex};
let bytes = sidecar_bytes();
let footer_at = bytes.len() - SIDX_FOOTER_LEN;
let (at, count, crc) = parse_footer(&bytes[footer_at..], bytes.len() as u64).unwrap();
let records = parse_table(&bytes[at as usize..footer_at], count, crc, at).unwrap();
let mut page_index = PageIndex::derive(&bytes, &records, &header()).unwrap();
let GroupSamples::Hash(samples) = &mut page_index.index.samples else {
panic!("hash fixture")
};
samples[0] += 1;
let mut sections: Vec<_> = records
.iter()
.map(|r| {
(
r.kind,
bytes[r.offset as usize..(r.offset + r.len) as usize].to_vec(),
)
})
.collect();
sections.push((SECTION_PAGE_INDEX, page_index.encode()));
let bytes = encode_sidecar(&header(), §ions);
assert!(matches!(
Sidecar::parse(&bytes),
Err(FormatFault::Corrupt(_))
));
}
#[test]
fn a_sidecar_without_a_page_index_still_parses() {
use crate::seg::{seal::sidx_name, segment::Segment, testkit};
let dir = tempfile::tempdir().unwrap();
for rec in [
testkit::write_legacy_segment(
crate::seg::fs::real(),
dir.path(),
1,
&testkit::entries(3_000, 1),
),
testkit::write_legacy_key_segment(dir.path(), &testkit::key_entries()),
] {
let bytes =
std::fs::read(dir.path().join(sidx_name(rec.segment_id, rec.version))).unwrap();
let sidecar = Sidecar::parse(&bytes).unwrap();
assert!(sidecar.page_index.is_none());
assert_eq!(rec.sidx_len, bytes.len() as u64);
assert_eq!(rec.sidx_checksum, xxhash_rust::xxh3::xxh3_64(&bytes));
let segment = Segment::open(
crate::seg::fs::real(),
dir.path(),
&rec,
testkit::KEY_LEN as u16,
testkit::SEED,
&std::sync::Arc::new(crate::seg::core::SegShared::new()),
)
.unwrap();
let mut report = crate::seg::verify::VerifyReport::default();
crate::seg::verify::verify_segment(&segment, &mut report).unwrap();
assert!(report.is_clean(), "{:?}", report.problems);
}
}
#[test]
fn rebuilt_crcs_do_not_hide_a_malformed_page_index() {
use crate::seg::{seal::sidx_name, testkit};
for ordered in [false, true] {
for mutation in [0, 1, 2] {
let dir = tempfile::tempdir().unwrap();
let rec = if ordered {
testkit::write_key_segment(dir.path(), &testkit::key_entries())
} else {
testkit::write_segment(
crate::seg::fs::real(),
dir.path(),
1,
&testkit::entries(3_000, 1),
)
};
let rec = testkit::rewrite_section(dir.path(), &rec, SECTION_PAGE_INDEX, |body| {
match mutation {
0 => {
let at = body.len() - 1;
body[at] ^= 1;
}
1 => body[48] ^= 1, _ => {
body.pop();
}
}
});
let bytes =
std::fs::read(dir.path().join(sidx_name(rec.segment_id, rec.version))).unwrap();
assert!(
matches!(Sidecar::parse(&bytes), Err(FormatFault::Corrupt(_))),
"ordered={ordered}, mutation={mutation}"
);
}
}
}
#[test]
fn duplicate_page_index_sections_are_corrupt() {
let bytes = sidecar_bytes();
let footer_at = bytes.len() - SIDX_FOOTER_LEN;
let (at, count, crc) = parse_footer(&bytes[footer_at..], bytes.len() as u64).unwrap();
let records = parse_table(&bytes[at as usize..footer_at], count, crc, at).unwrap();
let page_index = PageIndex::derive(&bytes, &records, &header()).unwrap();
let mut sections: Vec<_> = records
.iter()
.map(|r| {
(
r.kind,
bytes[r.offset as usize..(r.offset + r.len) as usize].to_vec(),
)
})
.collect();
sections.push((SECTION_PAGE_INDEX, page_index.encode()));
sections.push((SECTION_PAGE_INDEX, page_index.encode()));
assert!(matches!(
Sidecar::parse(&encode_sidecar(&header(), §ions)),
Err(FormatFault::Corrupt(_))
));
}
#[test]
fn paged_sidecar_writer_returns_an_internal_error_for_invalid_metadata() {
assert!(matches!(
encode_sidecar_paged(
&header(),
&[
(SECTION_BLOCK_INDEX, vec![0]),
(SECTION_FILTER, filter_section()),
(SECTION_PROPERTIES, props().encode()),
]
),
Err(DbError::Internal("segment writer: page index"))
));
}
fn header() -> SegHeader {
SegHeader {
segment_id: 0x2a,
version: 1,
order: OrderKind::ByHash,
key_len: 32,
seed: 0x1122_3344_5566_7788,
block_size: 16 * 1024,
}
}
fn props() -> Properties {
Properties {
entries: 3,
tombstones: 1,
min_gsn: 10,
max_gsn: 12,
min_h: 100,
max_h: 300,
raw_bytes: 456,
opened_at_ms: 1_000,
frozen_at_ms: 2_000,
sealed_at_ms: 3_000,
typ_hash: 0xabcd,
value_version: 7,
writer: "armdb test".into(),
}
}
fn index_section() -> Vec<u8> {
let mut b = BlockIndexBuilder::new();
b.push(100, 200, 2);
b.push(300, 150, 1);
b.finish()
}
fn filter_section() -> Vec<u8> {
let mut b = PartitionedFilterBuilder::new(0);
for h in [100u64, 200, 300] {
b.push(h).unwrap();
}
b.finish().unwrap()
}
fn sidecar_bytes() -> Vec<u8> {
encode_sidecar(
&header(),
&[
(SECTION_BLOCK_INDEX, index_section()),
(SECTION_FILTER, filter_section()),
(SECTION_PROPERTIES, props().encode()),
],
)
}
fn key_sections() -> Vec<(u32, Vec<u8>)> {
let mut b = crate::seg::key_index::SeparatorIndexBuilder::new();
b.push(b"", 100, 3).unwrap();
vec![
(SECTION_SEPARATOR_INDEX, b.finish().unwrap()),
(SECTION_KEY_RANGE, vec![0; 64]),
(SECTION_FILTER, filter_section()),
(SECTION_PROPERTIES, props().encode()),
]
}
#[test]
fn sidecar_sections_must_match_order() {
let mut h = header();
h.order = OrderKind::ByKey;
let sections = key_sections();
let parsed = Sidecar::parse(&encode_sidecar(&h, §ions)).unwrap();
assert!(matches!(
parsed.index,
crate::seg::key_index::SegmentIndex::Key(_)
));
assert!(parsed.key_range.is_some());
assert!(Sidecar::parse(&encode_sidecar(&header(), §ions)).is_err());
assert!(
Sidecar::parse(&encode_sidecar(
&h,
&[
(SECTION_BLOCK_INDEX, index_section()),
(SECTION_FILTER, filter_section()),
(SECTION_PROPERTIES, props().encode())
]
))
.is_err()
);
for missing in 0..sections.len() {
let mut bad = sections.clone();
bad.remove(missing);
assert!(Sidecar::parse(&encode_sidecar(&h, &bad)).is_err());
let mut bad = sections.clone();
bad.push(sections[missing].clone());
assert!(Sidecar::parse(&encode_sidecar(&h, &bad)).is_err());
}
}
#[test]
fn prefix_sections_are_crc_checked_and_unique() {
let mut h = header();
h.order = OrderKind::ByKey;
let mut prefix = vec![16, 0, 0, 0, 0, 0, 0, 0];
prefix.extend(filter_section());
let mut sections = key_sections();
sections.push((SECTION_PREFIX_FILTER, prefix.clone()));
let good = encode_sidecar(&h, §ions);
let parsed = Sidecar::parse(&good).unwrap();
assert_eq!(parsed.prefix_filters[0].0, 16);
assert_eq!(prefix[..8], [16, 0, 0, 0, 0, 0, 0, 0]);
let mut other = prefix.clone();
other[0] = 8;
sections.push((SECTION_PREFIX_FILTER, other));
assert_eq!(
Sidecar::parse(&encode_sidecar(&h, §ions))
.unwrap()
.prefix_filters
.len(),
2
);
sections.pop();
sections.push((SECTION_PREFIX_FILTER, prefix.clone()));
assert!(Sidecar::parse(&encode_sidecar(&h, §ions)).is_err());
sections.pop();
for (at, value) in [(0, 0), (0, 33), (2, 1)] {
let mut bad = prefix.clone();
bad[at] = value;
*sections.last_mut().unwrap() = (SECTION_PREFIX_FILTER, bad);
assert!(Sidecar::parse(&encode_sidecar(&h, §ions)).is_err());
}
for cut in 0..8 {
*sections.last_mut().unwrap() = (SECTION_PREFIX_FILTER, prefix[..cut].to_vec());
assert!(Sidecar::parse(&encode_sidecar(&h, §ions)).is_err());
}
let prefix_at = FILE_HEADER_LEN
+ key_sections()
.iter()
.map(|(_, body)| body.len())
.sum::<usize>();
let mut corrupt = good;
corrupt[prefix_at] ^= 1;
assert!(Sidecar::parse(&corrupt).is_err());
let bytes = encode_sidecar(
&header(),
&[
(SECTION_BLOCK_INDEX, index_section()),
(SECTION_FILTER, filter_section()),
(SECTION_PROPERTIES, props().encode()),
(SECTION_PREFIX_FILTER, vec![255]),
],
);
assert!(Sidecar::parse(&bytes).unwrap().prefix_filters.is_empty());
let prefix_at = FILE_HEADER_LEN
+ index_section().len()
+ filter_section().len()
+ props().encode().len();
let mut bad = bytes;
bad[prefix_at] ^= 1;
assert!(matches!(Sidecar::parse(&bad),Err(FormatFault::Corrupt(m)) if m.contains("CRC")));
}
#[test]
fn the_file_header_has_the_documented_layout() {
let bytes = header().encode(SDAT_MAGIC);
assert_eq!(&bytes[0..4], b"ASD1");
assert_eq!(u16::from_le_bytes([bytes[4], bytes[5]]), 1);
assert_eq!(bytes[6], NATIVE_BYTE_ORDER);
assert_eq!(bytes[7], 0);
assert_eq!(u64::from_le_bytes(bytes[8..16].try_into().unwrap()), 0x2a);
assert_eq!(u32::from_le_bytes(bytes[16..20].try_into().unwrap()), 1);
assert_eq!(u16::from_le_bytes([bytes[20], bytes[21]]), 32);
assert_eq!(
u64::from_le_bytes(bytes[24..32].try_into().unwrap()),
0x1122_3344_5566_7788
);
assert_eq!(
u32::from_le_bytes(bytes[32..36].try_into().unwrap()),
16 * 1024
);
assert!(bytes[36..60].iter().all(|&b| b == 0));
assert_eq!(
u32::from_le_bytes(bytes[60..64].try_into().unwrap()),
crc32fast::hash(&bytes[..60])
);
assert_eq!(SegHeader::decode(&bytes, SDAT_MAGIC).unwrap(), header());
}
fn reseal(bytes: &mut [u8; 64]) {
let crc = crc32fast::hash(&bytes[..60]);
bytes[60..64].copy_from_slice(&crc.to_le_bytes());
}
#[test]
fn the_file_header_fails_closed() {
let good = header().encode(SIDX_MAGIC);
assert!(matches!(
SegHeader::decode(&good, SDAT_MAGIC),
Err(FormatFault::Corrupt(m)) if m.contains("magic")
));
assert!(matches!(
SegHeader::decode(&good[..63], SIDX_MAGIC),
Err(FormatFault::Corrupt(_))
));
let mut flipped = good;
flipped[10] ^= 1;
assert!(matches!(
SegHeader::decode(&flipped, SIDX_MAGIC),
Err(FormatFault::Corrupt(m)) if m.contains("CRC")
));
let mut version = good;
version[4] = 2;
reseal(&mut version);
assert!(matches!(
SegHeader::decode(&version, SIDX_MAGIC),
Err(FormatFault::Unsupported(m)) if m.contains("version")
));
let mut byte_order = good;
byte_order[6] = 3 - NATIVE_BYTE_ORDER;
reseal(&mut byte_order);
assert!(matches!(
SegHeader::decode(&byte_order, SIDX_MAGIC),
Err(FormatFault::Unsupported(m)) if m.contains("byte order")
));
let mut by_key = good;
by_key[7] = 1;
reseal(&mut by_key);
assert_eq!(
SegHeader::decode(&by_key, SIDX_MAGIC).unwrap().order,
OrderKind::ByKey
);
let mut reserved = good;
reserved[40] = 1;
reseal(&mut reserved);
assert!(matches!(
SegHeader::decode(&reserved, SIDX_MAGIC),
Err(FormatFault::Unsupported(_))
));
}
#[test]
fn the_block_index_locates_blocks_by_hash() {
let index = BlockIndex::parse(&index_section()).unwrap();
assert_eq!(index.len(), 2);
assert_eq!(index.locate(0), None);
assert_eq!(index.locate(99), None);
assert_eq!(index.locate(100), Some(0));
assert_eq!(index.locate(299), Some(0));
assert_eq!(index.locate(300), Some(1));
assert_eq!(index.locate(u64::MAX), Some(1));
assert_eq!(index.range(0), (64, 200));
assert_eq!(index.range(1), (264, 150));
assert_eq!(index.end_offset(), 414);
assert_eq!(index.first_h(1), 300);
assert_eq!(index.resident_bytes(), 2 * 16 + 8);
}
#[test]
fn an_empty_block_index_is_valid() {
let section = BlockIndexBuilder::new().finish();
let index = BlockIndex::parse(§ion).unwrap();
assert!(index.is_empty());
assert_eq!(index.locate(5), None);
assert_eq!(index.end_offset(), 64);
}
#[test]
fn a_damaged_block_index_is_rejected() {
let good = index_section();
assert!(matches!(
BlockIndex::parse(&good[..good.len() - 1]),
Err(FormatFault::Corrupt(_))
));
assert!(matches!(
BlockIndex::parse(&[]),
Err(FormatFault::Corrupt(_))
));
let mut unordered = good.clone();
unordered[24..32].copy_from_slice(&100u64.to_le_bytes());
assert!(matches!(
BlockIndex::parse(&unordered),
Err(FormatFault::Corrupt(m)) if m.contains("order")
));
let mut gap = good.clone();
gap[32..40].copy_from_slice(&265u64.to_le_bytes());
assert!(matches!(
BlockIndex::parse(&gap),
Err(FormatFault::Corrupt(m)) if m.contains("contiguous")
));
let mut early = good.clone();
early[8..16].copy_from_slice(&10u64.to_le_bytes());
assert!(matches!(
BlockIndex::parse(&early),
Err(FormatFault::Corrupt(_))
));
let mut hollow = good.clone();
hollow[20..24].copy_from_slice(&0u32.to_le_bytes());
assert!(matches!(
BlockIndex::parse(&hollow),
Err(FormatFault::Corrupt(_))
));
let mut leading_gap = good.clone();
leading_gap[8..16].copy_from_slice(&65u64.to_le_bytes());
leading_gap[32..40].copy_from_slice(&265u64.to_le_bytes());
leading_gap[good.len() - 8..].copy_from_slice(&415u64.to_le_bytes());
assert!(matches!(
BlockIndex::parse(&leading_gap),
Err(FormatFault::Corrupt(_))
));
}
#[test]
fn properties_round_trip() {
let p = props();
assert_eq!(Properties::decode(&p.encode()).unwrap(), p);
let bytes = p.encode();
assert!(matches!(
Properties::decode(&bytes[..bytes.len() - 1]),
Err(FormatFault::Corrupt(_))
));
assert!(matches!(
Properties::decode(&bytes[..20]),
Err(FormatFault::Corrupt(_))
));
}
#[test]
fn a_sidecar_round_trips() {
let sidecar = Sidecar::parse(&sidecar_bytes()).unwrap();
assert_eq!(sidecar.header, header());
assert_eq!(sidecar.props, props());
let SegmentIndex::Hash(index) = sidecar.index else {
panic!("expected hash index");
};
assert_eq!(index.len(), 2);
assert!(sidecar.filter.may_contain(200));
}
#[test]
fn an_unknown_optional_section_is_skipped() {
let bytes = encode_sidecar(
&header(),
&[
(SECTION_BLOCK_INDEX, index_section()),
(OPTIONAL_SECTION_MIN + 5, b"from the future".to_vec()),
(SECTION_FILTER, filter_section()),
(SECTION_PROPERTIES, props().encode()),
],
);
assert!(Sidecar::parse(&bytes).is_ok());
}
#[test]
fn an_unknown_required_section_fails_the_parse() {
let bytes = encode_sidecar(
&header(),
&[
(SECTION_BLOCK_INDEX, index_section()),
(SECTION_FILTER, filter_section()),
(SECTION_PROPERTIES, props().encode()),
(9, b"required and unknown".to_vec()),
],
);
assert!(matches!(
Sidecar::parse(&bytes),
Err(FormatFault::Unsupported(m)) if m.contains("section kind 9")
));
}
#[test]
fn a_missing_required_section_fails_the_parse() {
let bytes = encode_sidecar(
&header(),
&[
(SECTION_BLOCK_INDEX, index_section()),
(SECTION_PROPERTIES, props().encode()),
],
);
assert!(matches!(
Sidecar::parse(&bytes),
Err(FormatFault::Corrupt(m)) if m.contains("filter")
));
}
fn with_first_record(mut bytes: Vec<u8>, edit: impl FnOnce(&mut [u8])) -> Vec<u8> {
let footer_at = bytes.len() - SIDX_FOOTER_LEN;
let count = u32::from_le_bytes(bytes[footer_at..footer_at + 4].try_into().unwrap());
let table_at = footer_at - count as usize * SECTION_RECORD_LEN;
edit(&mut bytes[table_at..table_at + SECTION_RECORD_LEN]);
let crc = crc32fast::hash(&bytes[table_at..footer_at]);
bytes[footer_at + 4..footer_at + 8].copy_from_slice(&crc.to_le_bytes());
bytes
}
#[test]
fn a_section_table_that_points_outside_the_file_is_rejected() {
let good = sidecar_bytes();
let past_end = with_first_record(good.clone(), |rec| {
rec[8..16].copy_from_slice(&u64::MAX.to_le_bytes());
});
assert!(matches!(
Sidecar::parse(&past_end),
Err(FormatFault::Corrupt(_))
));
let huge = with_first_record(good.clone(), |rec| {
rec[16..24].copy_from_slice(&u64::MAX.to_le_bytes());
});
assert!(matches!(
Sidecar::parse(&huge),
Err(FormatFault::Corrupt(_))
));
let in_header = with_first_record(good.clone(), |rec| {
rec[8..16].copy_from_slice(&8u64.to_le_bytes());
});
assert!(matches!(
Sidecar::parse(&in_header),
Err(FormatFault::Corrupt(m)) if m.contains("header")
));
let second_offset = {
let footer_at = good.len() - SIDX_FOOTER_LEN;
let table_at = footer_at - 3 * SECTION_RECORD_LEN;
let rec = &good[table_at + SECTION_RECORD_LEN..table_at + 2 * SECTION_RECORD_LEN];
u64::from_le_bytes(rec[8..16].try_into().unwrap())
};
let overlapping = with_first_record(good.clone(), |rec| {
rec[8..16].copy_from_slice(&second_offset.to_le_bytes());
});
assert!(matches!(
Sidecar::parse(&overlapping),
Err(FormatFault::Corrupt(_))
));
}
#[test]
fn a_damaged_tail_is_rejected() {
let good = sidecar_bytes();
for cut in [0, 10, FILE_HEADER_LEN, good.len() - 1] {
assert!(
matches!(Sidecar::parse(&good[..cut]), Err(FormatFault::Corrupt(_))),
"cut at {cut}"
);
}
let mut magic = good.clone();
let at = magic.len() - 8;
magic[at] = b'Z';
assert!(matches!(
Sidecar::parse(&magic),
Err(FormatFault::Corrupt(m)) if m.contains("footer")
));
let mut table = good.clone();
let at = table.len() - SIDX_FOOTER_LEN - 1;
table[at] ^= 1;
assert!(matches!(
Sidecar::parse(&table),
Err(FormatFault::Corrupt(m)) if m.contains("table")
));
let mut section = good.clone();
section[FILE_HEADER_LEN + 3] ^= 1;
assert!(matches!(
Sidecar::parse(§ion),
Err(FormatFault::Corrupt(m)) if m.contains("section")
));
}
#[test]
fn reserved_section_table_fields_are_rejected() {
for field in [4, 28] {
let bytes = with_first_record(sidecar_bytes(), |rec| rec[field] = 1);
assert!(matches!(
Sidecar::parse(&bytes),
Err(FormatFault::Unsupported(m)) if m.contains("reserved")
));
}
}
#[test]
fn duplicate_required_sections_are_rejected() {
let bytes = encode_sidecar(
&header(),
&[
(SECTION_BLOCK_INDEX, index_section()),
(SECTION_FILTER, filter_section()),
(SECTION_PROPERTIES, props().encode()),
(SECTION_PROPERTIES, props().encode()),
],
);
assert!(matches!(
Sidecar::parse(&bytes),
Err(FormatFault::Corrupt(_))
));
}
#[test]
fn damaged_optional_sections_are_rejected() {
let mut bytes = encode_sidecar(
&header(),
&[
(SECTION_BLOCK_INDEX, index_section()),
(OPTIONAL_SECTION_MIN, b"future".to_vec()),
(SECTION_FILTER, filter_section()),
(SECTION_PROPERTIES, props().encode()),
],
);
let at = FILE_HEADER_LEN + index_section().len();
bytes[at] ^= 1;
assert!(matches!(
Sidecar::parse(&bytes),
Err(FormatFault::Corrupt(_))
));
}
#[test]
fn set_footer_reserved_bytes_are_damage() {
let mut bytes = sidecar_bytes();
let at = bytes.len() - 1;
bytes[at] = 1;
assert!(matches!(
Sidecar::parse(&bytes),
Err(FormatFault::Corrupt(m)) if m.contains("footer reserved")
));
}
#[cfg(target_pointer_width = "64")]
#[test]
fn a_block_index_may_not_list_more_than_u32_max_blocks() {
assert!(check_block_count(0).is_ok());
assert!(check_block_count(u32::MAX as usize).is_ok());
assert!(matches!(
check_block_count(u32::MAX as usize + 1),
Err(FormatFault::Corrupt(m)) if m.contains("u32::MAX")
));
}
#[test]
fn the_section_table_and_footer_have_the_documented_layout() {
#[rustfmt::skip]
const RECORD: [u8; 32] = [
0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x48, 0x83, 0x9e, 0x00, 0x00, 0x00, 0x00, ];
#[rustfmt::skip]
const FOOTER: [u8; 24] = [
0x01, 0x00, 0x00, 0x00, 0x18, 0x76, 0x10, 0x9c, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x41, 0x53, 0x58, 0x46, 0x00, 0x00, 0x00, 0x00, ];
let bytes = encode_sidecar(&header(), &[(OPTIONAL_SECTION_MIN, b"ab".to_vec())]);
assert_eq!(bytes.len(), FILE_HEADER_LEN + 2 + 32 + 24);
assert_eq!(&bytes[FILE_HEADER_LEN..FILE_HEADER_LEN + 2], b"ab");
assert_eq!(bytes[66..98], RECORD);
assert_eq!(bytes[98..], FOOTER);
assert_eq!(parse_footer(&FOOTER, 122).unwrap(), (66, 1, 0x9c10_7618));
assert_eq!(
parse_table(&RECORD, 1, 0x9c10_7618, 66).unwrap(),
[SectionRecord {
kind: OPTIONAL_SECTION_MIN,
offset: 64,
len: 2,
crc32: 0x9e83_486d,
}]
);
}
#[test]
fn the_block_index_section_has_the_documented_layout() {
#[rustfmt::skip]
const INDEX: [u8; 56] = [
0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0xc8, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
0x2c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x08, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x96, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
0x9e, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
];
assert_eq!(index_section(), INDEX);
let (records, end) = parse_index_records(&INDEX).unwrap();
assert_eq!(
records,
[
BlockIndexRecord {
first_h: 100,
offset: 64,
block_len: 200,
entries: 2,
},
BlockIndexRecord {
first_h: 300,
offset: 264,
block_len: 150,
entries: 1,
},
]
);
assert_eq!(end, 414);
}
#[test]
fn the_properties_section_has_the_documented_layout() {
#[rustfmt::skip]
const PROPS: [u8; 102] = [
0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2c, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x0b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, 0xab, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x00, 0x0a, 0x00, b'a', b'r', b'm', b'd', b'b', b' ', b't', b'e', b's', b't',
];
assert_eq!(props().encode(), PROPS);
assert_eq!(Properties::decode(&PROPS).unwrap(), props());
}
}