use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::io::Read;
use sha2::{Digest, Sha256};
use crate::compression::{
decompress_exact_zstd_frame, decompress_exact_zstd_frame_with_dictionary,
validate_exact_zstd_frame,
};
use crate::crypto::{decrypt_padded_aead_object, verify_hmac, HmacDomain, MasterKey, Subkeys};
use crate::fec::repair_data_gf16;
use crate::format::{
BlockKind, FormatError, BLOCK_RECORD_FRAMING_LEN, BOOTSTRAP_SIDECAR_HEADER_LEN,
MANIFEST_FOOTER_LEN, VOLUME_HEADER_LEN, VOLUME_TRAILER_LEN,
};
use crate::metadata::{
hash_prefix, normalize_lookup_file_path, DirectoryHintShardEntry, DirectoryHintTable,
EnvelopeEntry, FileEntry, FrameEntry, IndexRoot, IndexShard, MetadataLimits, ShardEntry,
};
use crate::tar_model::{
parse_tar_member_group, restore_tar_member, validate_tar_stream_total_extraction_size,
MetadataDiagnostic, OwnedTarMember, SafeExtractionOptions, TarEntryKind,
};
use crate::wire::{
BlockRecord, BootstrapSidecarHeader, CryptoHeader, CryptoHeaderFixed, ManifestFooter,
VolumeHeader, VolumeTrailer,
};
const TRAILER_HMAC_COVERED_LEN: usize = 96;
const MANIFEST_HMAC_COVERED_LEN: usize = 104;
const SIDECAR_HMAC_COVERED_LEN: usize = 92;
const DEFAULT_MAX_VERIFY_TAR_SIZE: usize = 128 * 1024 * 1024;
const DEFAULT_MAX_TRAILING_GARBAGE_SCAN: usize = 1024 * 1024;
const DEFAULT_MAX_TOTAL_EXTRACTION_SIZE: u64 = 100 * 1024 * 1024 * 1024;
const DIRECTORY_HINT_REQUIRED_FILE_COUNT: u64 = 100_000;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ReaderOptions {
pub max_trailing_garbage_scan: usize,
pub max_verify_tar_size: usize,
pub max_total_extraction_size: u64,
}
impl Default for ReaderOptions {
fn default() -> Self {
Self {
max_trailing_garbage_scan: DEFAULT_MAX_TRAILING_GARBAGE_SCAN,
max_verify_tar_size: DEFAULT_MAX_VERIFY_TAR_SIZE,
max_total_extraction_size: DEFAULT_MAX_TOTAL_EXTRACTION_SIZE,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ArchiveEntry {
pub path: String,
pub file_data_size: u64,
pub kind: TarEntryKind,
pub mode: u32,
pub mtime: u64,
pub diagnostics: Vec<MetadataDiagnostic>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ExtractedArchiveMember {
pub path: String,
pub kind: TarEntryKind,
pub data: Vec<u8>,
pub link_target: Option<String>,
pub diagnostics: Vec<MetadataDiagnostic>,
}
#[derive(Debug, Clone)]
pub struct OpenedArchive {
options: ReaderOptions,
observed_archive_bytes: u64,
subkeys: Subkeys,
blocks: BTreeMap<u64, BlockRecord>,
pub volume_header: VolumeHeader,
pub crypto_header: CryptoHeaderFixed,
pub manifest_footer: ManifestFooter,
pub volume_trailer: VolumeTrailer,
pub index_root: IndexRoot,
payload_dictionary: Option<Vec<u8>>,
}
#[derive(Debug, Clone, Copy)]
struct ObjectExtent {
first_block_index: u64,
data_block_count: u32,
parity_block_count: u32,
encrypted_size: u32,
}
type DirectoryHintMap = BTreeMap<Vec<u8>, BTreeSet<u32>>;
pub fn open_archive<'a>(
bytes: &'a [u8],
master_key: &MasterKey,
) -> Result<OpenedArchive, FormatError> {
OpenedArchive::open_with_options(bytes, master_key, ReaderOptions::default())
}
pub fn open_archive_volumes(
volumes: &[&[u8]],
master_key: &MasterKey,
) -> Result<OpenedArchive, FormatError> {
OpenedArchive::open_volumes_with_options(volumes, master_key, ReaderOptions::default())
}
pub fn open_archive_with_bootstrap_sidecar(
bytes: &[u8],
bootstrap_sidecar: &[u8],
master_key: &MasterKey,
) -> Result<OpenedArchive, FormatError> {
OpenedArchive::open_with_bootstrap_sidecar_options(
bytes,
bootstrap_sidecar,
master_key,
ReaderOptions::default(),
)
}
pub fn open_non_seekable_archive(
bytes: &[u8],
master_key: &MasterKey,
bootstrap_sidecar: Option<&[u8]>,
) -> Result<OpenedArchive, FormatError> {
match bootstrap_sidecar {
Some(sidecar) => open_archive_with_bootstrap_sidecar(bytes, sidecar, master_key),
None => Err(FormatError::ReaderUnsupported(
"non-seekable random access requires a bootstrap sidecar",
)),
}
}
pub fn sequential_extract_tar_stream(
bytes: &[u8],
master_key: &MasterKey,
) -> Result<Vec<u8>, FormatError> {
sequential_extract_tar_stream_with_options(bytes, master_key, ReaderOptions::default())
}
impl OpenedArchive {
pub fn open_with_options(
bytes: &[u8],
master_key: &MasterKey,
options: ReaderOptions,
) -> Result<Self, FormatError> {
Self::open_volumes_with_options(&[bytes], master_key, options)
}
pub fn open_volumes_with_options(
volumes: &[&[u8]],
master_key: &MasterKey,
options: ReaderOptions,
) -> Result<Self, FormatError> {
if volumes.is_empty() {
return Err(FormatError::InvalidArchive("no volumes supplied"));
}
let observed_archive_bytes =
observed_archive_size(volumes.iter().map(|volume| volume.len() as u64))?;
let mut first: Option<ParsedSeekableVolume> = None;
let mut seen_volume_indexes = BTreeSet::new();
let mut blocks = BTreeMap::new();
let mut erased_block_indices = BTreeSet::new();
for volume_bytes in volumes {
let parsed = parse_seekable_volume(volume_bytes, master_key, options)?;
if !seen_volume_indexes.insert(parsed.volume_header.volume_index) {
return Err(FormatError::InvalidArchive(
"duplicate authenticated volume index",
));
}
if let Some(first) = &first {
validate_volume_set_member(first, &parsed)?;
}
for (block_index, record) in &parsed.blocks {
if blocks.insert(*block_index, record.clone()).is_some() {
return Err(FormatError::InvalidArchive("duplicate BlockRecord index"));
}
}
for block_index in &parsed.erased_block_indices {
erased_block_indices.insert(*block_index);
}
if first.is_none() {
first = Some(parsed);
}
}
let first = first.ok_or(FormatError::InvalidArchive("no volumes supplied"))?;
if seen_volume_indexes.len() == first.crypto_header.stripe_width as usize {
validate_complete_global_block_coverage(&blocks, &erased_block_indices)?;
}
let limits = metadata_limits(&first.crypto_header);
let index_root_plaintext = load_metadata_object_from_parts(
&blocks,
&first.subkeys,
&first.volume_header,
&first.crypto_header,
ObjectExtent {
first_block_index: first.manifest_footer.index_root_first_block,
data_block_count: first.manifest_footer.index_root_data_block_count,
parity_block_count: first.manifest_footer.index_root_parity_block_count,
encrypted_size: first.manifest_footer.index_root_encrypted_size,
},
BlockKind::IndexRootData,
BlockKind::IndexRootParity,
&first.subkeys.index_root_key,
&first.subkeys.index_nonce_seed,
b"idxroot",
0,
first.crypto_header.index_root_fec_data_shards,
first.crypto_header.index_root_fec_parity_shards,
first.manifest_footer.index_root_decompressed_size,
)?;
let index_root = IndexRoot::parse(
&index_root_plaintext,
first.crypto_header.has_dictionary != 0,
limits,
)?;
let payload_dictionary = load_archive_dictionary(
&blocks,
&first.subkeys,
&first.volume_header,
&first.crypto_header,
&index_root,
)?;
Ok(Self {
options,
observed_archive_bytes,
subkeys: first.subkeys,
blocks,
volume_header: first.volume_header,
crypto_header: first.crypto_header,
manifest_footer: first.manifest_footer,
volume_trailer: first.volume_trailer,
index_root,
payload_dictionary,
})
}
pub fn open_with_bootstrap_sidecar_options(
bytes: &[u8],
bootstrap_sidecar: &[u8],
master_key: &MasterKey,
options: ReaderOptions,
) -> Result<Self, FormatError> {
let observed_archive_bytes =
observed_archive_size([bytes.len() as u64, bootstrap_sidecar.len() as u64])?;
if bytes.len() < VOLUME_HEADER_LEN {
return Err(FormatError::InvalidLength {
structure: "archive",
expected: VOLUME_HEADER_LEN,
actual: bytes.len(),
});
}
let volume_header = VolumeHeader::parse(slice(bytes, 0, VOLUME_HEADER_LEN, "archive")?)?;
let crypto_start = volume_header.crypto_header_offset as usize;
let crypto_len = volume_header.crypto_header_length as usize;
let crypto_end = checked_add(crypto_start, crypto_len, "CryptoHeader")?;
let crypto_bytes = slice(bytes, crypto_start, crypto_len, "CryptoHeader")?;
let parsed_crypto = CryptoHeader::parse(crypto_bytes, volume_header.crypto_header_length)?;
let subkeys = Subkeys::derive(
master_key,
&volume_header.archive_uuid,
&volume_header.session_id,
)?;
verify_hmac(
HmacDomain::CryptoHeader,
&subkeys.mac_key,
&volume_header.archive_uuid,
&volume_header.session_id,
parsed_crypto.hmac_covered_bytes,
&parsed_crypto.header_hmac,
)?;
parsed_crypto.validate_extension_semantics()?;
validate_m9_supported_volume(&volume_header, &parsed_crypto.fixed)?;
let sidecar = parse_trusted_bootstrap_sidecar(
bootstrap_sidecar,
&volume_header,
&parsed_crypto.fixed,
&subkeys,
)?;
let (mut blocks, terminal_offset, observed_block_count) = parse_stream_block_prefix(
bytes,
crypto_end,
parsed_crypto.fixed.block_size as usize,
&volume_header,
)?;
insert_sidecar_records(&mut blocks, sidecar.index_root_records)?;
let (terminal_manifest, volume_trailer) = parse_terminal_material(
bytes,
terminal_offset,
observed_block_count,
&subkeys,
&volume_header,
parsed_crypto.fixed.block_size,
)?;
if terminal_manifest != sidecar.manifest_footer {
return Err(FormatError::InvalidArchive(
"bootstrap sidecar conflicts with terminal ManifestFooter",
));
}
let limits = metadata_limits(&parsed_crypto.fixed);
let index_root_plaintext = load_metadata_object_from_parts(
&blocks,
&subkeys,
&volume_header,
&parsed_crypto.fixed,
ObjectExtent {
first_block_index: sidecar.manifest_footer.index_root_first_block,
data_block_count: sidecar.manifest_footer.index_root_data_block_count,
parity_block_count: sidecar.manifest_footer.index_root_parity_block_count,
encrypted_size: sidecar.manifest_footer.index_root_encrypted_size,
},
BlockKind::IndexRootData,
BlockKind::IndexRootParity,
&subkeys.index_root_key,
&subkeys.index_nonce_seed,
b"idxroot",
0,
parsed_crypto.fixed.index_root_fec_data_shards,
parsed_crypto.fixed.index_root_fec_parity_shards,
sidecar.manifest_footer.index_root_decompressed_size,
)?;
let index_root = IndexRoot::parse(
&index_root_plaintext,
parsed_crypto.fixed.has_dictionary != 0,
limits,
)?;
if parsed_crypto.fixed.has_dictionary != 0 {
let (offset, length) =
sidecar
.dictionary_records_section
.ok_or(FormatError::ReaderUnsupported(
"dictionary bootstrap required",
))?;
let dictionary_records = parse_sidecar_block_records(
bootstrap_sidecar,
offset,
length,
parsed_crypto.fixed.block_size as usize,
dictionary_extent_from_index_root(&index_root)?,
BlockKind::DictionaryData,
BlockKind::DictionaryParity,
"dictionary",
)?;
insert_sidecar_records(&mut blocks, dictionary_records)?;
}
let payload_dictionary = load_archive_dictionary(
&blocks,
&subkeys,
&volume_header,
&parsed_crypto.fixed,
&index_root,
)?;
Ok(Self {
options,
observed_archive_bytes,
subkeys,
blocks,
volume_header,
crypto_header: parsed_crypto.fixed,
manifest_footer: sidecar.manifest_footer,
volume_trailer,
index_root,
payload_dictionary,
})
}
pub fn list_files(&self) -> Result<Vec<ArchiveEntry>, FormatError> {
#[derive(Clone, Copy)]
struct WinningEntry {
start: u64,
file_data_size: u64,
shard_index: usize,
file_index: usize,
}
let shards = self.load_all_index_shards()?;
let mut final_entries = BTreeMap::<String, WinningEntry>::new();
for (shard_index, shard) in shards.iter().enumerate() {
for (idx, file) in shard.files.iter().enumerate() {
let path = utf8_path(
shard
.file_path(idx)
.ok_or(FormatError::InvalidArchive("FileEntry path is missing"))?,
)?;
let start =
shard
.tar_member_group_start(idx)
.ok_or(FormatError::InvalidArchive(
"FileEntry tar member start is missing",
))?;
if let Some(winner) = final_entries.get_mut(&path) {
if start >= winner.start {
winner.start = start;
winner.file_data_size = file.file_data_size;
winner.shard_index = shard_index;
winner.file_index = idx;
}
} else {
final_entries.insert(
path,
WinningEntry {
start,
file_data_size: file.file_data_size,
shard_index,
file_index: idx,
},
);
}
}
}
final_entries
.into_iter()
.map(|(path, winner)| {
let shard = &shards[winner.shard_index];
let member =
self.decode_loaded_owned_tar_member(shard, winner.file_index, false)?;
Ok(ArchiveEntry {
path,
file_data_size: winner.file_data_size,
kind: member.kind,
mode: member.mode,
mtime: member.mtime,
diagnostics: member.diagnostics,
})
})
.collect()
}
pub fn extract_file(&self, path: &str) -> Result<Option<Vec<u8>>, FormatError> {
self.extract_member(path)?
.map(|member| {
if member.kind != TarEntryKind::Regular {
return Err(FormatError::ReaderUnsupported(
"extract_file returns only regular file payloads",
));
}
Ok(member.data)
})
.transpose()
}
pub fn extract_member(
&self,
path: &str,
) -> Result<Option<ExtractedArchiveMember>, FormatError> {
let normalized = normalize_lookup_file_path(path, self.crypto_header.max_path_length)?;
let candidate_indexes = self
.index_root
.candidate_shards_for_path(&normalized, self.metadata_limits())?;
let mut winner: Option<(IndexShard, usize, u64)> = None;
for row_index in candidate_indexes {
let locating =
self.index_root
.shards
.get(row_index)
.ok_or(FormatError::InvalidArchive(
"candidate shard row is out of bounds",
))?;
let shard = self.load_index_shard(locating)?;
if let Some(file_index) = shard.lookup_file_index(&normalized) {
let start =
shard
.tar_member_group_start(file_index)
.ok_or(FormatError::InvalidArchive(
"FileEntry tar member start is missing",
))?;
if winner
.as_ref()
.map(|(_, _, best_start)| start > *best_start)
.unwrap_or(true)
{
winner = Some((shard, file_index, start));
}
}
}
winner
.map(|(shard, file_index, _)| self.extract_loaded_member(&shard, file_index))
.transpose()
}
pub fn extract_file_to(
&self,
path: &str,
root: &std::path::Path,
options: SafeExtractionOptions,
) -> Result<Option<Vec<MetadataDiagnostic>>, FormatError> {
self.extract_owned_tar_member(path)?
.map(|member| restore_tar_member(root, &member, options))
.transpose()
}
pub fn verify(&self) -> Result<(), FormatError> {
let shards = self.load_all_index_shards()?;
let mut file_count = 0u64;
let mut frames = BTreeMap::<u64, FrameEntry>::new();
let mut envelopes = BTreeMap::<u64, EnvelopeEntry>::new();
for shard in &shards {
file_count = file_count
.checked_add(shard.files.len() as u64)
.ok_or(FormatError::InvalidArchive("file count overflow"))?;
for frame in &shard.frames {
if let Some(existing) = frames.insert(frame.frame_index, frame.clone()) {
if existing != *frame {
return Err(FormatError::InvalidArchive(
"duplicate FrameEntry rows do not match",
));
}
}
}
for envelope in &shard.envelopes {
if let Some(existing) = envelopes.insert(envelope.envelope_index, envelope.clone())
{
if existing != *envelope {
return Err(FormatError::InvalidArchive(
"duplicate EnvelopeEntry rows do not match",
));
}
}
}
}
if file_count != self.index_root.header.file_count {
return Err(FormatError::InvalidArchive(
"IndexRoot file_count does not match decoded shards",
));
}
if self.index_root.header.file_count > DIRECTORY_HINT_REQUIRED_FILE_COUNT
&& self.index_root.directory_hint_shards.is_empty()
{
return Err(FormatError::InvalidArchive(
"IndexRoot file_count requires directory hints",
));
}
verify_dense_keys(&frames, self.index_root.header.frame_count, "FrameEntry")?;
verify_dense_keys(
&envelopes,
self.index_root.header.envelope_count,
"EnvelopeEntry",
)?;
validate_envelope_frame_coverage(&frames, &envelopes)?;
self.validate_encrypted_object_block_ranges(&envelopes)?;
let payload_block_count = envelopes.values().try_fold(0u64, |sum, envelope| {
sum.checked_add(envelope.data_block_count as u64)
.ok_or(FormatError::InvalidArchive("payload block count overflow"))
})?;
if payload_block_count != self.index_root.header.payload_block_count {
return Err(FormatError::InvalidArchive(
"IndexRoot payload_block_count does not match envelopes",
));
}
let tar_len = to_usize(self.index_root.header.tar_total_size, "tar stream")?;
if tar_len > self.options.max_verify_tar_size {
return Err(FormatError::ReaderUnsupported(
"verify tar stream exceeds configured in-memory cap",
));
}
let mut tar_stream = vec![0u8; tar_len];
let mut covered = vec![false; tar_len];
let mut envelope_cache = HashMap::<u64, Vec<u8>>::new();
for frame in frames.values() {
let envelope =
envelopes
.get(&frame.envelope_index)
.ok_or(FormatError::InvalidArchive(
"FrameEntry references missing EnvelopeEntry",
))?;
if !envelope_cache.contains_key(&envelope.envelope_index) {
envelope_cache.insert(
envelope.envelope_index,
self.load_payload_envelope(envelope)?,
);
}
let envelope_plaintext = envelope_cache
.get(&envelope.envelope_index)
.expect("inserted above");
let compressed = slice(
envelope_plaintext,
frame.offset_in_envelope as usize,
frame.compressed_size as usize,
"FrameEntry",
)?;
let decoded = self.decompress_payload_frame(compressed, frame.decompressed_size)?;
let start = to_usize(frame.tar_stream_offset, "tar stream")?;
let end = checked_add(start, decoded.len(), "tar stream")?;
if end > tar_stream.len() {
return Err(FormatError::InvalidArchive(
"FrameEntry exceeds IndexRoot tar_total_size",
));
}
if covered[start..end].iter().any(|value| *value) {
return Err(FormatError::InvalidArchive("decoded frames overlap"));
}
tar_stream[start..end].copy_from_slice(&decoded);
covered[start..end].fill(true);
}
if covered.iter().any(|value| !*value) {
return Err(FormatError::InvalidArchive("decoded frames leave tar gap"));
}
if sha256_bytes(&tar_stream) != self.index_root.header.content_sha256 {
return Err(FormatError::InvalidArchive(
"IndexRoot content_sha256 does not match decoded tar stream",
));
}
let mut file_extents = Vec::new();
let mut directory_hint_map = DirectoryHintMap::new();
for (shard_row_index, shard) in shards.iter().enumerate() {
let shard_row_index = u32::try_from(shard_row_index)
.map_err(|_| FormatError::InvalidArchive("shard row index overflow"))?;
for idx in 0..shard.files.len() {
let file = &shard.files[idx];
let start =
shard
.tar_member_group_start(idx)
.ok_or(FormatError::InvalidArchive(
"FileEntry tar member start is missing",
))?;
file_extents.push((start, file.tar_member_group_size));
let member = self.decode_loaded_owned_tar_member(shard, idx, false)?;
let path = shard
.file_path(idx)
.ok_or(FormatError::InvalidArchive("FileEntry path is missing"))?;
add_expected_directory_hint_rows(
&mut directory_hint_map,
shard_row_index,
path,
member.kind,
);
}
}
validate_file_extent_coverage_ranges(&file_extents, tar_len)?;
if !self.index_root.directory_hint_shards.is_empty() {
let hint_tables = self.load_all_directory_hint_tables()?;
validate_directory_hint_tables_against_expected(&hint_tables, &directory_hint_map)?;
}
Ok(())
}
fn load_all_index_shards(&self) -> Result<Vec<IndexShard>, FormatError> {
self.index_root
.shards
.iter()
.map(|entry| self.load_index_shard(entry))
.collect()
}
fn load_index_shard(&self, entry: &ShardEntry) -> Result<IndexShard, FormatError> {
let plaintext = load_metadata_object_from_parts(
&self.blocks,
&self.subkeys,
&self.volume_header,
&self.crypto_header,
ObjectExtent {
first_block_index: entry.first_block_index,
data_block_count: entry.data_block_count,
parity_block_count: entry.parity_block_count,
encrypted_size: entry.encrypted_size,
},
BlockKind::IndexShardData,
BlockKind::IndexShardParity,
&self.subkeys.index_shard_key,
&self.subkeys.index_nonce_seed,
b"idxshard",
entry.shard_index,
self.crypto_header.index_fec_data_shards,
self.crypto_header.index_fec_parity_shards,
entry.decompressed_size,
)?;
IndexShard::parse(&plaintext, entry, self.metadata_limits())
}
fn load_all_directory_hint_tables(&self) -> Result<Vec<DirectoryHintTable>, FormatError> {
self.index_root
.directory_hint_shards
.iter()
.map(|entry| self.load_directory_hint_table(entry))
.collect()
}
fn load_directory_hint_table(
&self,
entry: &DirectoryHintShardEntry,
) -> Result<DirectoryHintTable, FormatError> {
let plaintext = load_metadata_object_from_parts(
&self.blocks,
&self.subkeys,
&self.volume_header,
&self.crypto_header,
ObjectExtent {
first_block_index: entry.first_block_index,
data_block_count: entry.data_block_count,
parity_block_count: entry.parity_block_count,
encrypted_size: entry.encrypted_size,
},
BlockKind::DirectoryHintData,
BlockKind::DirectoryHintParity,
&self.subkeys.dir_hint_key,
&self.subkeys.index_nonce_seed,
b"dirhint",
entry.hint_shard_index,
self.crypto_header.index_fec_data_shards,
self.crypto_header.index_fec_parity_shards,
entry.decompressed_size,
)?;
DirectoryHintTable::parse(
&plaintext,
entry,
self.index_root.header.shard_count,
self.metadata_limits(),
)
}
fn load_payload_envelope(&self, envelope: &EnvelopeEntry) -> Result<Vec<u8>, FormatError> {
let plaintext = load_decrypted_object_from_parts(
&self.blocks,
&self.volume_header,
&self.crypto_header,
ObjectExtent {
first_block_index: envelope.first_block_index,
data_block_count: envelope.data_block_count,
parity_block_count: envelope.parity_block_count,
encrypted_size: envelope.encrypted_size,
},
BlockKind::PayloadData,
BlockKind::PayloadParity,
&self.subkeys.enc_key,
&self.subkeys.nonce_seed,
b"envelope",
envelope.envelope_index,
self.crypto_header.fec_data_shards,
self.crypto_header.fec_parity_shards,
)?;
if plaintext.len() != envelope.plaintext_size as usize {
return Err(FormatError::InvalidArchive(
"payload envelope plaintext_size mismatch",
));
}
Ok(plaintext)
}
fn extract_owned_tar_member(&self, path: &str) -> Result<Option<OwnedTarMember>, FormatError> {
let normalized = normalize_lookup_file_path(path, self.crypto_header.max_path_length)?;
let candidate_indexes = self
.index_root
.candidate_shards_for_path(&normalized, self.metadata_limits())?;
let mut winner: Option<(IndexShard, usize, u64)> = None;
for row_index in candidate_indexes {
let locating =
self.index_root
.shards
.get(row_index)
.ok_or(FormatError::InvalidArchive(
"candidate shard row is out of bounds",
))?;
let shard = self.load_index_shard(locating)?;
if let Some(file_index) = shard.lookup_file_index(&normalized) {
let start =
shard
.tar_member_group_start(file_index)
.ok_or(FormatError::InvalidArchive(
"FileEntry tar member start is missing",
))?;
if winner
.as_ref()
.map(|(_, _, best_start)| start > *best_start)
.unwrap_or(true)
{
winner = Some((shard, file_index, start));
}
}
}
winner
.map(|(shard, file_index, _)| self.extract_loaded_owned_tar_member(&shard, file_index))
.transpose()
}
fn extract_loaded_member(
&self,
shard: &IndexShard,
file_index: usize,
) -> Result<ExtractedArchiveMember, FormatError> {
let member = self.extract_loaded_owned_tar_member(shard, file_index)?;
Ok(ExtractedArchiveMember {
path: utf8_path(&member.path)?,
kind: member.kind,
data: member.data,
link_target: member
.link_target
.map(|target| utf8_path(&target))
.transpose()?,
diagnostics: member.diagnostics,
})
}
fn extract_loaded_owned_tar_member(
&self,
shard: &IndexShard,
file_index: usize,
) -> Result<OwnedTarMember, FormatError> {
self.decode_loaded_owned_tar_member(shard, file_index, true)
}
fn decode_loaded_owned_tar_member(
&self,
shard: &IndexShard,
file_index: usize,
enforce_extraction_cap: bool,
) -> Result<OwnedTarMember, FormatError> {
let file = shard
.files
.get(file_index)
.ok_or(FormatError::InvalidArchive("FileEntry index out of bounds"))?;
if enforce_extraction_cap {
self.validate_total_extraction_size(file.file_data_size)?;
}
let expected_path = shard
.file_path(file_index)
.ok_or(FormatError::InvalidArchive("FileEntry path is missing"))?;
let frames = frame_range_for_file(shard, file)?;
let mut envelope_cache = HashMap::<u64, Vec<u8>>::new();
let mut decoded = Vec::new();
for frame in frames {
let envelope = shard
.envelopes
.iter()
.find(|entry| entry.envelope_index == frame.envelope_index)
.ok_or(FormatError::InvalidArchive(
"FrameEntry references missing EnvelopeEntry",
))?;
if !envelope_cache.contains_key(&envelope.envelope_index) {
envelope_cache.insert(
envelope.envelope_index,
self.load_payload_envelope(envelope)?,
);
}
let envelope_plaintext = envelope_cache
.get(&envelope.envelope_index)
.expect("inserted above");
let compressed = slice(
envelope_plaintext,
frame.offset_in_envelope as usize,
frame.compressed_size as usize,
"FrameEntry",
)?;
decoded.extend_from_slice(
&self.decompress_payload_frame(compressed, frame.decompressed_size)?,
);
}
let offset = file.offset_in_first_frame_plaintext as usize;
let group_len = to_usize(file.tar_member_group_size, "FileEntry")?;
let group = slice(&decoded, offset, group_len, "FileEntry")?;
let member = parse_tar_member_group(group, self.crypto_header.max_path_length)?;
if member.path != expected_path {
return Err(FormatError::InvalidArchive(
"tar member path does not match FileEntry path",
));
}
if member.logical_size != file.file_data_size {
return Err(FormatError::InvalidArchive(
"tar member size does not match FileEntry file_data_size",
));
}
Ok(member.to_owned_member())
}
fn metadata_limits(&self) -> MetadataLimits {
metadata_limits(&self.crypto_header)
}
fn validate_total_extraction_size(&self, logical_size: u64) -> Result<(), FormatError> {
let cap = total_extraction_size_cap(self.options, self.observed_archive_bytes);
if logical_size > cap {
return Err(FormatError::ReaderUnsupported(
"total extraction size exceeds configured cap",
));
}
Ok(())
}
fn decompress_payload_frame(
&self,
compressed: &[u8],
decompressed_size: u32,
) -> Result<Vec<u8>, FormatError> {
if let Some(dictionary) = &self.payload_dictionary {
decompress_exact_zstd_frame_with_dictionary(
compressed,
decompressed_size as usize,
dictionary,
)
} else {
decompress_exact_zstd_frame(compressed, decompressed_size as usize)
}
}
fn validate_encrypted_object_block_ranges(
&self,
envelopes: &BTreeMap<u64, EnvelopeEntry>,
) -> Result<(), FormatError> {
let mut ranges = Vec::new();
ranges.push(object_block_range(
self.manifest_footer.index_root_first_block,
self.manifest_footer.index_root_data_block_count,
self.manifest_footer.index_root_parity_block_count,
"IndexRoot",
)?);
for shard in &self.index_root.shards {
ranges.push(object_block_range(
shard.first_block_index,
shard.data_block_count,
shard.parity_block_count,
"IndexShard",
)?);
}
for hint in &self.index_root.directory_hint_shards {
ranges.push(object_block_range(
hint.first_block_index,
hint.data_block_count,
hint.parity_block_count,
"DirectoryHintShardEntry",
)?);
}
if self.crypto_header.has_dictionary != 0 {
ranges.push(object_block_range(
self.index_root.header.dictionary_first_block,
self.index_root.header.dictionary_data_block_count,
self.index_root.header.dictionary_parity_block_count,
"dictionary",
)?);
}
for envelope in envelopes.values() {
ranges.push(object_block_range(
envelope.first_block_index,
envelope.data_block_count,
envelope.parity_block_count,
"EnvelopeEntry",
)?);
}
validate_non_overlapping_object_ranges(&mut ranges)
}
}
#[derive(Debug)]
struct ParsedSeekableVolume {
volume_header: VolumeHeader,
crypto_header: CryptoHeaderFixed,
crypto_header_bytes: Vec<u8>,
subkeys: Subkeys,
manifest_footer: ManifestFooter,
volume_trailer: VolumeTrailer,
blocks: BTreeMap<u64, BlockRecord>,
erased_block_indices: BTreeSet<u64>,
}
fn parse_seekable_volume(
bytes: &[u8],
master_key: &MasterKey,
options: ReaderOptions,
) -> Result<ParsedSeekableVolume, FormatError> {
if bytes.len() < VOLUME_HEADER_LEN + VOLUME_TRAILER_LEN {
return Err(FormatError::InvalidLength {
structure: "archive",
expected: VOLUME_HEADER_LEN + VOLUME_TRAILER_LEN,
actual: bytes.len(),
});
}
let volume_header = VolumeHeader::parse(slice(bytes, 0, VOLUME_HEADER_LEN, "archive")?)?;
let crypto_start = volume_header.crypto_header_offset as usize;
let crypto_len = volume_header.crypto_header_length as usize;
let crypto_end = checked_add(crypto_start, crypto_len, "CryptoHeader")?;
let crypto_bytes = slice(bytes, crypto_start, crypto_len, "CryptoHeader")?;
let parsed_crypto = CryptoHeader::parse(crypto_bytes, volume_header.crypto_header_length)?;
let subkeys = Subkeys::derive(
master_key,
&volume_header.archive_uuid,
&volume_header.session_id,
)?;
verify_hmac(
HmacDomain::CryptoHeader,
&subkeys.mac_key,
&volume_header.archive_uuid,
&volume_header.session_id,
parsed_crypto.hmac_covered_bytes,
&parsed_crypto.header_hmac,
)?;
parsed_crypto.validate_extension_semantics()?;
validate_seekable_supported_volume(&volume_header, &parsed_crypto.fixed)?;
let (trailer_offset, volume_trailer) =
locate_trailer(bytes, &subkeys, &volume_header, options)?;
validate_trailer_identity(&volume_header, &volume_trailer)?;
let manifest_offset = to_usize(volume_trailer.manifest_footer_offset, "ManifestFooter")?;
let manifest_end = checked_add(manifest_offset, MANIFEST_FOOTER_LEN, "ManifestFooter")?;
if manifest_end != trailer_offset {
return Err(FormatError::InvalidArchive(
"ManifestFooter does not end at selected trailer",
));
}
let manifest_bytes = slice(
bytes,
manifest_offset,
MANIFEST_FOOTER_LEN,
"ManifestFooter",
)?;
let manifest_footer = ManifestFooter::parse(manifest_bytes)?;
validate_manifest_footer(&volume_header, &manifest_footer, &subkeys, manifest_bytes)?;
manifest_footer.validate_index_root_extent(parsed_crypto.fixed.block_size)?;
let block_region = parse_block_region(
bytes,
crypto_end,
manifest_offset,
parsed_crypto.fixed.block_size as usize,
&volume_header,
&volume_trailer,
)?;
Ok(ParsedSeekableVolume {
volume_header,
crypto_header: parsed_crypto.fixed,
crypto_header_bytes: crypto_bytes.to_vec(),
subkeys,
manifest_footer,
volume_trailer,
blocks: block_region.blocks,
erased_block_indices: block_region.erased_block_indices,
})
}
fn validate_seekable_supported_volume(
volume_header: &VolumeHeader,
crypto_header: &CryptoHeaderFixed,
) -> Result<(), FormatError> {
if crypto_header.stripe_width != volume_header.stripe_width {
return Err(FormatError::InvalidArchive(
"VolumeHeader and CryptoHeader stripe_width differ",
));
}
Ok(())
}
fn validate_volume_set_member(
first: &ParsedSeekableVolume,
candidate: &ParsedSeekableVolume,
) -> Result<(), FormatError> {
if candidate.volume_header.archive_uuid != first.volume_header.archive_uuid
|| candidate.volume_header.session_id != first.volume_header.session_id
{
return Err(FormatError::InvalidArchive(
"mixed archive or session IDs in volume set",
));
}
if candidate.crypto_header_bytes != first.crypto_header_bytes
|| candidate.crypto_header != first.crypto_header
{
return Err(FormatError::InvalidArchive("CryptoHeader copies differ"));
}
if !manifest_bootstrap_fields_match(&first.manifest_footer, &candidate.manifest_footer) {
return Err(FormatError::InvalidArchive(
"ManifestFooter bootstrap fields differ",
));
}
Ok(())
}
fn manifest_bootstrap_fields_match(left: &ManifestFooter, right: &ManifestFooter) -> bool {
left.archive_uuid == right.archive_uuid
&& left.session_id == right.session_id
&& left.is_authoritative == right.is_authoritative
&& left.total_volumes == right.total_volumes
&& left.index_root_first_block == right.index_root_first_block
&& left.index_root_data_block_count == right.index_root_data_block_count
&& left.index_root_parity_block_count == right.index_root_parity_block_count
&& left.index_root_encrypted_size == right.index_root_encrypted_size
&& left.index_root_decompressed_size == right.index_root_decompressed_size
}
fn validate_complete_global_block_coverage(
blocks: &BTreeMap<u64, BlockRecord>,
erased_block_indices: &BTreeSet<u64>,
) -> Result<(), FormatError> {
let mut expected = 0u64;
let mut block_iter = blocks.keys().copied().peekable();
let mut erasure_iter = erased_block_indices.iter().copied().peekable();
loop {
let next_block = block_iter.peek().copied();
let next_erasure = erasure_iter.peek().copied();
let next = match (next_block, next_erasure) {
(Some(block), Some(erasure)) if block == erasure => {
return Err(FormatError::InvalidArchive(
"BlockRecord index is both present and erased",
));
}
(Some(block), Some(erasure)) => block.min(erasure),
(Some(block), None) => block,
(None, Some(erasure)) => erasure,
(None, None) => return Ok(()),
};
if next != expected {
return Err(FormatError::InvalidArchive(
"complete volume set has missing global blocks",
));
}
if next_block == Some(next) {
block_iter.next();
}
if next_erasure == Some(next) {
erasure_iter.next();
}
expected = expected
.checked_add(1)
.ok_or(FormatError::InvalidArchive("global block index overflow"))?;
}
}
fn locate_trailer(
bytes: &[u8],
subkeys: &Subkeys,
volume_header: &VolumeHeader,
options: ReaderOptions,
) -> Result<(usize, VolumeTrailer), FormatError> {
let canonical_offset =
bytes
.len()
.checked_sub(VOLUME_TRAILER_LEN)
.ok_or(FormatError::InvalidLength {
structure: "VolumeTrailer",
expected: VOLUME_TRAILER_LEN,
actual: bytes.len(),
})?;
match parse_authenticated_trailer(bytes, canonical_offset, subkeys, volume_header) {
Ok(trailer) => {
if trailer.bytes_written != canonical_offset as u64 {
return Err(FormatError::InvalidArchive(
"VolumeTrailer bytes_written does not match selected trailer offset",
));
}
return Ok((canonical_offset, trailer));
}
Err(err) if options.max_trailing_garbage_scan == 0 => return Err(err),
Err(_) => {}
}
let scan_start = canonical_offset.saturating_sub(options.max_trailing_garbage_scan);
for offset in (scan_start..canonical_offset).rev() {
if let Ok(trailer) = parse_authenticated_trailer(bytes, offset, subkeys, volume_header) {
if trailer.bytes_written == offset as u64 {
return Ok((offset, trailer));
}
}
}
Err(FormatError::InvalidArchive(
"no authenticated VolumeTrailer found",
))
}
fn parse_authenticated_trailer(
bytes: &[u8],
offset: usize,
subkeys: &Subkeys,
volume_header: &VolumeHeader,
) -> Result<VolumeTrailer, FormatError> {
let raw = slice(bytes, offset, VOLUME_TRAILER_LEN, "VolumeTrailer")?;
let trailer = VolumeTrailer::parse(raw)?;
verify_hmac(
HmacDomain::VolumeTrailer,
&subkeys.mac_key,
&volume_header.archive_uuid,
&volume_header.session_id,
&raw[..TRAILER_HMAC_COVERED_LEN],
&trailer.trailer_hmac,
)?;
Ok(trailer)
}
fn validate_m9_supported_volume(
volume_header: &VolumeHeader,
crypto_header: &CryptoHeaderFixed,
) -> Result<(), FormatError> {
if volume_header.stripe_width != 1 || volume_header.volume_index != 0 {
return Err(FormatError::ReaderUnsupported(
"M9 reader supports only single-volume archives",
));
}
if crypto_header.stripe_width != volume_header.stripe_width {
return Err(FormatError::InvalidArchive(
"VolumeHeader and CryptoHeader stripe_width differ",
));
}
Ok(())
}
#[derive(Debug)]
struct TrustedBootstrapSidecar {
manifest_footer: ManifestFooter,
index_root_records: Vec<BlockRecord>,
dictionary_records_section: Option<(u64, u64)>,
}
fn parse_trusted_bootstrap_sidecar(
bytes: &[u8],
volume_header: &VolumeHeader,
crypto_header: &CryptoHeaderFixed,
subkeys: &Subkeys,
) -> Result<TrustedBootstrapSidecar, FormatError> {
let header_bytes = slice(
bytes,
0,
BOOTSTRAP_SIDECAR_HEADER_LEN,
"BootstrapSidecarHeader",
)?;
let header = BootstrapSidecarHeader::parse(header_bytes)?;
if header.archive_uuid != volume_header.archive_uuid
|| header.session_id != volume_header.session_id
{
return Err(FormatError::InvalidArchive(
"bootstrap sidecar identity does not match VolumeHeader",
));
}
verify_hmac(
HmacDomain::BootstrapSidecar,
&subkeys.mac_key,
&volume_header.archive_uuid,
&volume_header.session_id,
&header_bytes[..SIDECAR_HMAC_COVERED_LEN],
&header.sidecar_hmac,
)?;
header.validate_packed_layout(bytes.len() as u64)?;
validate_sidecar_size_cap(&header, crypto_header, bytes.len() as u64)?;
if !header.has_manifest_footer() || !header.has_index_root_records() {
return Err(FormatError::ReaderUnsupported(
"non-seekable bootstrap sidecar requires ManifestFooter and IndexRoot sections",
));
}
if header.has_dictionary_records() {
if crypto_header.has_dictionary == 0 {
return Err(FormatError::InvalidArchive(
"bootstrap sidecar has dictionary records while has_dictionary is false",
));
}
} else if crypto_header.has_dictionary != 0 {
return Err(FormatError::ReaderUnsupported(
"dictionary bootstrap required",
));
}
let manifest_offset = to_usize(header.manifest_footer_offset, "BootstrapSidecarHeader")?;
let manifest_bytes = slice(
bytes,
manifest_offset,
MANIFEST_FOOTER_LEN,
"ManifestFooter",
)?;
let manifest_footer = ManifestFooter::parse(manifest_bytes)?;
validate_sidecar_manifest_footer(
volume_header,
crypto_header,
&manifest_footer,
subkeys,
manifest_bytes,
)?;
manifest_footer.validate_index_root_extent(crypto_header.block_size)?;
let index_root_records = parse_sidecar_block_records(
bytes,
header.index_root_records_offset,
header.index_root_records_length,
crypto_header.block_size as usize,
ObjectExtent {
first_block_index: manifest_footer.index_root_first_block,
data_block_count: manifest_footer.index_root_data_block_count,
parity_block_count: manifest_footer.index_root_parity_block_count,
encrypted_size: manifest_footer.index_root_encrypted_size,
},
BlockKind::IndexRootData,
BlockKind::IndexRootParity,
"IndexRoot",
)?;
Ok(TrustedBootstrapSidecar {
manifest_footer,
index_root_records,
dictionary_records_section: header.has_dictionary_records().then_some((
header.dictionary_records_offset,
header.dictionary_records_length,
)),
})
}
fn insert_sidecar_records(
blocks: &mut BTreeMap<u64, BlockRecord>,
records: Vec<BlockRecord>,
) -> Result<(), FormatError> {
for record in records {
if let Some(existing) = blocks.insert(record.block_index, record.clone()) {
if existing != record {
return Err(FormatError::InvalidArchive(
"bootstrap sidecar conflicts with volume BlockRecord",
));
}
}
}
Ok(())
}
fn validate_sidecar_manifest_footer(
volume_header: &VolumeHeader,
crypto_header: &CryptoHeaderFixed,
footer: &ManifestFooter,
subkeys: &Subkeys,
raw: &[u8],
) -> Result<(), FormatError> {
if footer.archive_uuid != volume_header.archive_uuid
|| footer.session_id != volume_header.session_id
{
return Err(FormatError::InvalidArchive(
"sidecar ManifestFooter identity does not match VolumeHeader",
));
}
if footer.volume_index != 0 {
return Err(FormatError::InvalidArchive(
"sidecar ManifestFooter volume_index must be zero",
));
}
if footer.total_volumes != crypto_header.stripe_width {
return Err(FormatError::InvalidArchive(
"sidecar ManifestFooter total_volumes does not match stripe_width",
));
}
if footer.is_authoritative != 1 {
return Err(FormatError::InvalidArchive(
"sidecar ManifestFooter is not authoritative",
));
}
verify_hmac(
HmacDomain::ManifestFooter,
&subkeys.mac_key,
&volume_header.archive_uuid,
&volume_header.session_id,
&raw[..MANIFEST_HMAC_COVERED_LEN],
&footer.manifest_hmac,
)
}
fn validate_sidecar_size_cap(
header: &BootstrapSidecarHeader,
crypto_header: &CryptoHeaderFixed,
file_size: u64,
) -> Result<(), FormatError> {
let record_len = checked_u64_add(
crypto_header.block_size as u64,
BLOCK_RECORD_FRAMING_LEN as u64,
"bootstrap sidecar cap overflow",
)?;
let max_index_records = crypto_header.index_root_fec_data_shards as u64
+ crypto_header.index_root_fec_parity_shards as u64;
let max_record_section_bytes = checked_u64_mul(
max_index_records,
record_len,
"bootstrap sidecar cap overflow",
)?;
if header.index_root_records_length % record_len != 0 {
return Err(FormatError::InvalidArchive(
"bootstrap sidecar IndexRoot records length is not aligned",
));
}
if header.index_root_records_length / record_len > max_index_records {
return Err(FormatError::InvalidArchive(
"bootstrap sidecar IndexRoot records exceed resource cap",
));
}
if header.dictionary_records_length % record_len != 0 {
return Err(FormatError::InvalidArchive(
"bootstrap sidecar dictionary records length is not aligned",
));
}
if header.dictionary_records_length / record_len > max_index_records {
return Err(FormatError::InvalidArchive(
"bootstrap sidecar dictionary records exceed resource cap",
));
}
let mut cap = BOOTSTRAP_SIDECAR_HEADER_LEN as u64;
if header.has_manifest_footer() {
cap = cap
.checked_add(MANIFEST_FOOTER_LEN as u64)
.ok_or(FormatError::InvalidArchive(
"bootstrap sidecar cap overflow",
))?;
}
if header.has_index_root_records() {
cap = checked_u64_add(
cap,
max_record_section_bytes,
"bootstrap sidecar cap overflow",
)?;
}
if header.has_dictionary_records() {
cap = checked_u64_add(
cap,
max_record_section_bytes,
"bootstrap sidecar cap overflow",
)?;
}
if file_size > cap {
return Err(FormatError::InvalidArchive(
"bootstrap sidecar exceeds resource cap",
));
}
Ok(())
}
fn parse_sidecar_block_records(
sidecar_bytes: &[u8],
offset: u64,
length: u64,
block_size: usize,
extent: ObjectExtent,
data_kind: BlockKind,
parity_kind: BlockKind,
structure: &'static str,
) -> Result<Vec<BlockRecord>, FormatError> {
let record_len = block_size
.checked_add(BLOCK_RECORD_FRAMING_LEN)
.ok_or(FormatError::InvalidArchive("BlockRecord length overflow"))?;
if length % record_len as u64 != 0 {
return Err(FormatError::InvalidArchive(
"sidecar BlockRecord section is not aligned",
));
}
let expected_count = extent.data_block_count as usize + extent.parity_block_count as usize;
let actual_count = usize::try_from(length / record_len as u64)
.map_err(|_| FormatError::InvalidArchive("sidecar BlockRecord count overflow"))?;
if actual_count != expected_count {
return Err(FormatError::InvalidArchive(
"sidecar BlockRecord section does not match declared extent",
));
}
let start = to_usize(offset, "BootstrapSidecarHeader")?;
let raw = slice(
sidecar_bytes,
start,
to_usize(length, "BootstrapSidecarHeader")?,
"BootstrapSidecarHeader",
)?;
let mut records = Vec::with_capacity(expected_count);
for idx in 0..expected_count {
let record = BlockRecord::parse(
slice(raw, idx * record_len, record_len, "BlockRecord")?,
block_size,
)?;
let expected_block_index =
checked_u64_add(extent.first_block_index, idx as u64, structure)?;
if record.block_index != expected_block_index {
return Err(FormatError::InvalidArchive(
"sidecar BlockRecord section has missing or duplicate blocks",
));
}
let expected_kind = if idx < extent.data_block_count as usize {
data_kind
} else {
parity_kind
};
if record.kind != expected_kind {
return Err(FormatError::InvalidArchive(
"sidecar BlockRecord section has wrong kind",
));
}
let should_be_last = idx + 1 == extent.data_block_count as usize;
if idx < extent.data_block_count as usize && record.is_last_data() != should_be_last {
return Err(FormatError::InvalidArchive(
"sidecar BlockRecord section has wrong last-data flag",
));
}
records.push(record);
}
Ok(records)
}
fn validate_trailer_identity(
volume_header: &VolumeHeader,
trailer: &VolumeTrailer,
) -> Result<(), FormatError> {
if trailer.archive_uuid != volume_header.archive_uuid
|| trailer.session_id != volume_header.session_id
|| trailer.volume_index != volume_header.volume_index
{
return Err(FormatError::InvalidArchive(
"VolumeTrailer identity does not match VolumeHeader",
));
}
Ok(())
}
fn validate_manifest_footer(
volume_header: &VolumeHeader,
footer: &ManifestFooter,
subkeys: &Subkeys,
raw: &[u8],
) -> Result<(), FormatError> {
if footer.archive_uuid != volume_header.archive_uuid
|| footer.session_id != volume_header.session_id
|| footer.volume_index != volume_header.volume_index
{
return Err(FormatError::InvalidArchive(
"ManifestFooter identity does not match VolumeHeader",
));
}
if footer.total_volumes != volume_header.stripe_width {
return Err(FormatError::InvalidArchive(
"ManifestFooter total_volumes does not match stripe_width",
));
}
if footer.is_authoritative != 1 {
return Err(FormatError::InvalidArchive(
"ManifestFooter is not authoritative",
));
}
verify_hmac(
HmacDomain::ManifestFooter,
&subkeys.mac_key,
&volume_header.archive_uuid,
&volume_header.session_id,
&raw[..MANIFEST_HMAC_COVERED_LEN],
&footer.manifest_hmac,
)
}
#[derive(Debug)]
struct ParsedBlockRegion {
blocks: BTreeMap<u64, BlockRecord>,
erased_block_indices: BTreeSet<u64>,
}
fn parse_block_region(
bytes: &[u8],
start: usize,
end: usize,
block_size: usize,
volume_header: &VolumeHeader,
trailer: &VolumeTrailer,
) -> Result<ParsedBlockRegion, FormatError> {
if end < start {
return Err(FormatError::InvalidArchive(
"ManifestFooter starts before BlockRecord region",
));
}
let record_len = block_size
.checked_add(BLOCK_RECORD_FRAMING_LEN)
.ok_or(FormatError::InvalidArchive("BlockRecord length overflow"))?;
let region_len = end - start;
if region_len % record_len != 0 {
return Err(FormatError::InvalidArchive(
"BlockRecord region length is not aligned",
));
}
let observed_count = region_len / record_len;
if observed_count as u64 != trailer.block_count {
return Err(FormatError::InvalidArchive(
"VolumeTrailer block_count does not match BlockRecord region",
));
}
let mut blocks = BTreeMap::new();
let mut erased_block_indices = BTreeSet::new();
for idx in 0..observed_count {
let offset = start + idx * record_len;
let expected_block_index = checked_u64_add(
volume_header.volume_index as u64,
checked_u64_mul(
idx as u64,
volume_header.stripe_width as u64,
"BlockRecord index overflow",
)?,
"BlockRecord index overflow",
)?;
let raw = slice(bytes, offset, record_len, "BlockRecord")?;
match BlockRecord::parse(raw, block_size) {
Ok(record) => {
if record.block_index != expected_block_index {
return Err(FormatError::InvalidArchive(
"BlockRecord index does not match volume position",
));
}
if blocks.insert(record.block_index, record).is_some() {
return Err(FormatError::InvalidArchive("duplicate BlockRecord index"));
}
}
Err(err) if block_record_error_is_recoverable_erasure(&err) => {
if !erased_block_indices.insert(expected_block_index) {
return Err(FormatError::InvalidArchive(
"duplicate erased BlockRecord index",
));
}
}
Err(err) => return Err(err),
}
}
Ok(ParsedBlockRegion {
blocks,
erased_block_indices,
})
}
fn block_record_error_is_recoverable_erasure(error: &FormatError) -> bool {
match error {
FormatError::BadCrc { structure }
| FormatError::BadMagic { structure }
| FormatError::NonZeroReserved { structure } => *structure == "BlockRecord",
_ => false,
}
}
fn checked_u64_mul(lhs: u64, rhs: u64, reason: &'static str) -> Result<u64, FormatError> {
lhs.checked_mul(rhs)
.ok_or(FormatError::InvalidArchive(reason))
}
fn parse_stream_block_prefix(
bytes: &[u8],
start: usize,
block_size: usize,
volume_header: &VolumeHeader,
) -> Result<(BTreeMap<u64, BlockRecord>, usize, u64), FormatError> {
let record_len = block_size
.checked_add(BLOCK_RECORD_FRAMING_LEN)
.ok_or(FormatError::InvalidArchive("BlockRecord length overflow"))?;
let mut blocks = BTreeMap::new();
let mut offset = start;
let mut observed_block_count = 0u64;
while bytes.get(offset..offset + 4) == Some(b"TZBK") {
let expected_block_index =
expected_stream_block_index(volume_header, observed_block_count)?;
let raw = slice(bytes, offset, record_len, "BlockRecord")?;
match BlockRecord::parse(raw, block_size) {
Ok(record) => {
if record.block_index != expected_block_index {
return Err(FormatError::InvalidArchive(
"BlockRecord index does not match stream position",
));
}
if blocks.insert(record.block_index, record).is_some() {
return Err(FormatError::InvalidArchive("duplicate BlockRecord index"));
}
}
Err(err) if block_record_error_is_recoverable_erasure(&err) => {}
Err(err) => return Err(err),
}
offset = checked_add(offset, record_len, "BlockRecord")?;
observed_block_count = observed_block_count
.checked_add(1)
.ok_or(FormatError::InvalidArchive("BlockRecord count overflow"))?;
}
Ok((blocks, offset, observed_block_count))
}
fn expected_stream_block_index(
volume_header: &VolumeHeader,
observed_block_count: u64,
) -> Result<u64, FormatError> {
checked_u64_add(
volume_header.volume_index as u64,
checked_u64_mul(
observed_block_count,
volume_header.stripe_width as u64,
"BlockRecord index overflow",
)?,
"BlockRecord index overflow",
)
}
fn parse_sequential_block_or_erasure(
bytes: &[u8],
offset: usize,
record_len: usize,
block_size: usize,
volume_header: &VolumeHeader,
observed_block_count: u64,
) -> Result<Option<BlockRecord>, FormatError> {
let expected_block_index = expected_stream_block_index(volume_header, observed_block_count)?;
let raw = slice(bytes, offset, record_len, "BlockRecord")?;
match BlockRecord::parse(raw, block_size) {
Ok(record) => {
if record.block_index != expected_block_index {
return Err(FormatError::InvalidArchive(
"BlockRecord index does not match stream position",
));
}
Ok(Some(record))
}
Err(err) if block_record_error_is_recoverable_erasure(&err) => Ok(None),
Err(err) => Err(err),
}
}
fn parse_terminal_material(
bytes: &[u8],
manifest_offset: usize,
observed_block_count: u64,
subkeys: &Subkeys,
volume_header: &VolumeHeader,
block_size: u32,
) -> Result<(ManifestFooter, VolumeTrailer), FormatError> {
let manifest_end = checked_add(manifest_offset, MANIFEST_FOOTER_LEN, "ManifestFooter")?;
let trailer_end = checked_add(manifest_end, VOLUME_TRAILER_LEN, "VolumeTrailer")?;
if trailer_end != bytes.len() {
return Err(FormatError::InvalidArchive(
"terminal ManifestFooter/VolumeTrailer is not packed at stream end",
));
}
let manifest_bytes = slice(
bytes,
manifest_offset,
MANIFEST_FOOTER_LEN,
"ManifestFooter",
)?;
let manifest_footer = ManifestFooter::parse(manifest_bytes)?;
validate_manifest_footer(volume_header, &manifest_footer, subkeys, manifest_bytes)?;
manifest_footer.validate_index_root_extent(block_size)?;
let trailer = parse_authenticated_trailer(bytes, manifest_end, subkeys, volume_header)?;
validate_trailer_identity(volume_header, &trailer)?;
if trailer.manifest_footer_offset != manifest_offset as u64 {
return Err(FormatError::InvalidArchive(
"VolumeTrailer ManifestFooter offset does not match observed stream offset",
));
}
if trailer.manifest_footer_length != MANIFEST_FOOTER_LEN as u32 {
return Err(FormatError::InvalidManifestFooterLength(
trailer.manifest_footer_length,
));
}
if trailer.bytes_written != manifest_end as u64 {
return Err(FormatError::InvalidArchive(
"VolumeTrailer bytes_written does not match observed trailer offset",
));
}
if trailer.block_count != observed_block_count {
return Err(FormatError::InvalidArchive(
"VolumeTrailer block_count does not match observed stream",
));
}
Ok((manifest_footer, trailer))
}
#[derive(Debug, Default)]
struct PendingSequentialEnvelope {
data_shards: Vec<Option<Vec<u8>>>,
parity_shards: Vec<Option<Vec<u8>>>,
saw_last_data: bool,
awaiting_tentative_parity: bool,
}
impl PendingSequentialEnvelope {
fn is_empty(&self) -> bool {
self.data_shards.is_empty() && self.parity_shards.is_empty()
}
}
fn handle_sequential_payload_erasure(
pending: &mut PendingSequentialEnvelope,
crypto_header: &CryptoHeaderFixed,
metadata_seen: bool,
) -> Result<(), FormatError> {
if metadata_seen || pending.saw_last_data {
return Err(FormatError::BadCrc {
structure: "BlockRecord",
});
}
if !sequential_payload_parity_is_guaranteed(crypto_header) {
return Err(FormatError::BadCrc {
structure: "BlockRecord",
});
}
pending.data_shards.push(None);
pending.awaiting_tentative_parity = true;
if pending.data_shards.len() > crypto_header.fec_data_shards as usize {
return Err(FormatError::InvalidArchive(
"sequential payload envelope exceeds data-shard cap",
));
}
Ok(())
}
fn sequential_payload_parity_is_guaranteed(crypto_header: &CryptoHeaderFixed) -> bool {
crypto_header.fec_parity_shards > 0
&& (crypto_header.volume_loss_tolerance > 0 || crypto_header.bit_rot_buffer_pct > 0)
}
fn sequential_extract_tar_stream_with_options(
bytes: &[u8],
master_key: &MasterKey,
options: ReaderOptions,
) -> Result<Vec<u8>, FormatError> {
if bytes.len() < VOLUME_HEADER_LEN {
return Err(FormatError::InvalidLength {
structure: "archive",
expected: VOLUME_HEADER_LEN,
actual: bytes.len(),
});
}
let volume_header = VolumeHeader::parse(slice(bytes, 0, VOLUME_HEADER_LEN, "archive")?)?;
let crypto_start = volume_header.crypto_header_offset as usize;
let crypto_len = volume_header.crypto_header_length as usize;
let crypto_end = checked_add(crypto_start, crypto_len, "CryptoHeader")?;
let crypto_bytes = slice(bytes, crypto_start, crypto_len, "CryptoHeader")?;
let parsed_crypto = CryptoHeader::parse(crypto_bytes, volume_header.crypto_header_length)?;
let subkeys = Subkeys::derive(
master_key,
&volume_header.archive_uuid,
&volume_header.session_id,
)?;
verify_hmac(
HmacDomain::CryptoHeader,
&subkeys.mac_key,
&volume_header.archive_uuid,
&volume_header.session_id,
parsed_crypto.hmac_covered_bytes,
&parsed_crypto.header_hmac,
)?;
parsed_crypto.validate_extension_semantics()?;
validate_sequential_supported_volume(&volume_header, &parsed_crypto.fixed)?;
let block_size = parsed_crypto.fixed.block_size as usize;
let record_len = block_size
.checked_add(BLOCK_RECORD_FRAMING_LEN)
.ok_or(FormatError::InvalidArchive("BlockRecord length overflow"))?;
let mut offset = crypto_end;
let mut observed_block_count = 0u64;
let mut metadata_seen = false;
let mut pending = PendingSequentialEnvelope::default();
let mut next_envelope_index = 0u64;
let mut tar_stream = Vec::new();
while bytes.get(offset..offset + 4) == Some(b"TZBK") {
let record = parse_sequential_block_or_erasure(
bytes,
offset,
record_len,
block_size,
&volume_header,
observed_block_count,
)?;
observed_block_count = observed_block_count
.checked_add(1)
.ok_or(FormatError::InvalidArchive("BlockRecord count overflow"))?;
let Some(record) = record else {
handle_sequential_payload_erasure(&mut pending, &parsed_crypto.fixed, metadata_seen)?;
offset = checked_add(offset, record_len, "BlockRecord")?;
continue;
};
match record.kind {
BlockKind::PayloadData => {
if metadata_seen {
return Err(FormatError::InvalidArchive(
"payload BlockRecord appears after metadata",
));
}
if pending.awaiting_tentative_parity {
return Err(FormatError::InvalidArchive(
"sequential payload envelope boundary is ambiguous after CRC erasure",
));
}
if pending.saw_last_data {
finalize_sequential_envelope(
&mut pending,
&parsed_crypto.fixed,
&subkeys,
&volume_header,
&mut next_envelope_index,
&mut tar_stream,
)?;
}
let is_last_data = record.is_last_data();
pending.data_shards.push(Some(record.payload));
if is_last_data {
pending.saw_last_data = true;
}
if pending.data_shards.len() > parsed_crypto.fixed.fec_data_shards as usize {
return Err(FormatError::InvalidArchive(
"sequential payload envelope exceeds data-shard cap",
));
}
}
BlockKind::PayloadParity => {
if metadata_seen {
return Err(FormatError::InvalidArchive(
"payload parity BlockRecord appears after metadata",
));
}
if pending.awaiting_tentative_parity {
pending.awaiting_tentative_parity = false;
pending.saw_last_data = true;
} else if pending.data_shards.is_empty() || !pending.saw_last_data {
return Err(FormatError::InvalidArchive(
"payload parity appears before envelope data is complete",
));
}
pending.parity_shards.push(Some(record.payload));
if pending.parity_shards.len() > parsed_crypto.fixed.fec_parity_shards as usize {
return Err(FormatError::InvalidArchive(
"sequential payload envelope exceeds parity-shard cap",
));
}
}
_ => {
if !pending.is_empty() {
finalize_sequential_envelope(
&mut pending,
&parsed_crypto.fixed,
&subkeys,
&volume_header,
&mut next_envelope_index,
&mut tar_stream,
)?;
}
metadata_seen = true;
}
}
offset = checked_add(offset, record_len, "BlockRecord")?;
}
if !pending.is_empty() {
finalize_sequential_envelope(
&mut pending,
&parsed_crypto.fixed,
&subkeys,
&volume_header,
&mut next_envelope_index,
&mut tar_stream,
)?;
}
parse_terminal_material(
bytes,
offset,
observed_block_count,
&subkeys,
&volume_header,
parsed_crypto.fixed.block_size,
)?;
let observed_archive_bytes = observed_archive_size([bytes.len() as u64])?;
validate_tar_stream_total_extraction_size(
&tar_stream,
parsed_crypto.fixed.max_path_length,
total_extraction_size_cap(options, observed_archive_bytes),
)?;
Ok(tar_stream)
}
fn validate_sequential_supported_volume(
volume_header: &VolumeHeader,
crypto_header: &CryptoHeaderFixed,
) -> Result<(), FormatError> {
if volume_header.stripe_width != 1 || volume_header.volume_index != 0 {
return Err(FormatError::ReaderUnsupported(
"M9 sequential reader supports only single-volume archives",
));
}
if crypto_header.stripe_width != volume_header.stripe_width {
return Err(FormatError::InvalidArchive(
"VolumeHeader and CryptoHeader stripe_width differ",
));
}
if crypto_header.has_dictionary != 0 {
return Err(FormatError::ReaderUnsupported(
"dictionary bootstrap required for non-seekable sequential extraction",
));
}
Ok(())
}
fn finalize_sequential_envelope(
pending: &mut PendingSequentialEnvelope,
crypto_header: &CryptoHeaderFixed,
subkeys: &Subkeys,
volume_header: &VolumeHeader,
next_envelope_index: &mut u64,
tar_stream: &mut Vec<u8>,
) -> Result<(), FormatError> {
if !pending.saw_last_data {
return Err(FormatError::InvalidArchive(
"sequential payload envelope is missing last-data flag",
));
}
if pending.data_shards.len() > crypto_header.fec_data_shards as usize {
return Err(FormatError::InvalidArchive(
"sequential payload envelope exceeds data-shard cap",
));
}
if pending.parity_shards.len() > crypto_header.fec_parity_shards as usize {
return Err(FormatError::InvalidArchive(
"sequential payload envelope exceeds parity-shard cap",
));
}
let required_parity = required_object_parity(pending.data_shards.len() as u64, crypto_header)?;
if pending.parity_shards.len() < required_parity as usize {
return Err(FormatError::InvalidArchive(
"sequential payload envelope has insufficient parity for recovery settings",
));
}
let repaired = repair_data_gf16(
&pending.data_shards,
&pending.parity_shards,
crypto_header.block_size as usize,
)?;
let mut encrypted = Vec::with_capacity(repaired.len() * crypto_header.block_size as usize);
for shard in repaired {
encrypted.extend_from_slice(&shard);
}
let plaintext = decrypt_padded_aead_object(
crypto_header.aead_algo,
&subkeys.enc_key,
&subkeys.nonce_seed,
b"envelope",
&volume_header.archive_uuid,
&volume_header.session_id,
*next_envelope_index,
&encrypted,
)?;
decode_concatenated_zstd_frames(&plaintext, None, tar_stream)?;
*next_envelope_index = next_envelope_index
.checked_add(1)
.ok_or(FormatError::InvalidArchive("envelope counter overflow"))?;
*pending = PendingSequentialEnvelope::default();
Ok(())
}
fn decode_concatenated_zstd_frames(
plaintext: &[u8],
dictionary: Option<&[u8]>,
output: &mut Vec<u8>,
) -> Result<(), FormatError> {
let mut cursor = 0usize;
while cursor < plaintext.len() {
let frame_len = zstd_safe::find_frame_compressed_size(&plaintext[cursor..])
.map_err(|_| FormatError::InvalidZstdFrame)?;
if frame_len == 0 {
return Err(FormatError::InvalidZstdFrame);
}
let end = checked_add(cursor, frame_len, "zstd frame")?;
validate_exact_zstd_frame(&plaintext[cursor..end])?;
let decoded = if let Some(dictionary) = dictionary {
let mut decoder =
zstd::stream::Decoder::with_dictionary(&plaintext[cursor..end], dictionary)
.map_err(|_| FormatError::ZstdDecompressionFailure)?;
let mut decoded = Vec::new();
decoder
.read_to_end(&mut decoded)
.map_err(|_| FormatError::ZstdDecompressionFailure)?;
decoded
} else {
zstd::stream::decode_all(&plaintext[cursor..end])
.map_err(|_| FormatError::ZstdDecompressionFailure)?
};
output.extend_from_slice(&decoded);
cursor = end;
}
Ok(())
}
fn load_archive_dictionary(
blocks: &BTreeMap<u64, BlockRecord>,
subkeys: &Subkeys,
volume_header: &VolumeHeader,
crypto_header: &CryptoHeaderFixed,
index_root: &IndexRoot,
) -> Result<Option<Vec<u8>>, FormatError> {
if crypto_header.has_dictionary == 0 {
return Ok(None);
}
let plaintext = load_metadata_object_from_parts(
blocks,
subkeys,
volume_header,
crypto_header,
dictionary_extent_from_index_root(index_root)?,
BlockKind::DictionaryData,
BlockKind::DictionaryParity,
&subkeys.dictionary_key,
&subkeys.index_nonce_seed,
b"dict",
0,
crypto_header.index_root_fec_data_shards,
crypto_header.index_root_fec_parity_shards,
index_root.header.dictionary_decompressed_size,
)?;
Ok(Some(plaintext))
}
fn dictionary_extent_from_index_root(index_root: &IndexRoot) -> Result<ObjectExtent, FormatError> {
if index_root.header.dictionary_data_block_count == 0
|| index_root.header.dictionary_encrypted_size == 0
|| index_root.header.dictionary_decompressed_size == 0
{
return Err(FormatError::InvalidArchive("dictionary bootstrap required"));
}
Ok(ObjectExtent {
first_block_index: index_root.header.dictionary_first_block,
data_block_count: index_root.header.dictionary_data_block_count,
parity_block_count: index_root.header.dictionary_parity_block_count,
encrypted_size: index_root.header.dictionary_encrypted_size,
})
}
fn load_metadata_object_from_parts(
blocks: &BTreeMap<u64, BlockRecord>,
subkeys: &Subkeys,
volume_header: &VolumeHeader,
crypto_header: &CryptoHeaderFixed,
extent: ObjectExtent,
data_kind: BlockKind,
parity_kind: BlockKind,
key: &[u8; 32],
nonce_seed: &[u8; 32],
domain: &[u8],
counter: u64,
class_data_shard_max: u16,
class_parity_shard_max: u16,
decompressed_size: u32,
) -> Result<Vec<u8>, FormatError> {
let compressed = load_decrypted_object_from_parts(
blocks,
volume_header,
crypto_header,
extent,
data_kind,
parity_kind,
key,
nonce_seed,
domain,
counter,
class_data_shard_max,
class_parity_shard_max,
)?;
let _ = subkeys;
decompress_exact_zstd_frame(&compressed, decompressed_size as usize)
}
fn load_decrypted_object_from_parts(
blocks: &BTreeMap<u64, BlockRecord>,
volume_header: &VolumeHeader,
crypto_header: &CryptoHeaderFixed,
extent: ObjectExtent,
data_kind: BlockKind,
parity_kind: BlockKind,
key: &[u8; 32],
nonce_seed: &[u8; 32],
domain: &[u8],
counter: u64,
class_data_shard_max: u16,
class_parity_shard_max: u16,
) -> Result<Vec<u8>, FormatError> {
validate_object_extent(
extent,
crypto_header,
class_data_shard_max,
class_parity_shard_max,
)?;
let block_size = crypto_header.block_size as usize;
let data_count = extent.data_block_count as usize;
let parity_count = extent.parity_block_count as usize;
let mut data_shards = Vec::with_capacity(data_count);
let mut parity_shards = Vec::with_capacity(parity_count);
for offset in 0..data_count {
let block_index = checked_u64_add(extent.first_block_index, offset as u64, "object")?;
if let Some(record) = blocks.get(&block_index) {
if record.kind != data_kind {
return Err(FormatError::InvalidArchive(
"object data block has unexpected kind",
));
}
let should_be_last = offset + 1 == data_count;
if record.is_last_data() != should_be_last {
return Err(FormatError::InvalidArchive(
"object last-data flag is not on the final data block",
));
}
data_shards.push(Some(record.payload.clone()));
} else {
data_shards.push(None);
}
}
for offset in 0..parity_count {
let block_index = checked_u64_add(
extent.first_block_index,
data_count as u64 + offset as u64,
"object",
)?;
if let Some(record) = blocks.get(&block_index) {
if record.kind != parity_kind {
return Err(FormatError::InvalidArchive(
"object parity block has unexpected kind",
));
}
if record.is_last_data() {
return Err(FormatError::InvalidArchive(
"object parity block has last-data flag",
));
}
parity_shards.push(Some(record.payload.clone()));
} else {
parity_shards.push(None);
}
}
let repaired = repair_data_gf16(&data_shards, &parity_shards, block_size)?;
let mut encrypted = Vec::with_capacity(extent.encrypted_size as usize);
for shard in repaired {
encrypted.extend_from_slice(&shard);
}
if encrypted.len() != extent.encrypted_size as usize {
return Err(FormatError::InvalidArchive(
"object encrypted size does not match repaired shards",
));
}
decrypt_padded_aead_object(
crypto_header.aead_algo,
key,
nonce_seed,
domain,
&volume_header.archive_uuid,
&volume_header.session_id,
counter,
&encrypted,
)
}
fn validate_object_extent(
extent: ObjectExtent,
crypto_header: &CryptoHeaderFixed,
class_data_shard_max: u16,
class_parity_shard_max: u16,
) -> Result<(), FormatError> {
if extent.data_block_count == 0 || extent.encrypted_size == 0 {
return Err(FormatError::InvalidArchive(
"encrypted object has zero data blocks or size",
));
}
if extent.data_block_count > class_data_shard_max as u32 {
return Err(FormatError::InvalidArchive(
"encrypted object exceeds its class data-shard maximum",
));
}
if extent.parity_block_count > class_parity_shard_max as u32 {
return Err(FormatError::InvalidArchive(
"encrypted object exceeds its class parity-shard maximum",
));
}
let required_parity = required_object_parity(extent.data_block_count as u64, crypto_header)?;
if extent.parity_block_count < required_parity {
return Err(FormatError::InvalidArchive(
"encrypted object has insufficient parity for recovery settings",
));
}
let total = checked_u64_add(
extent.data_block_count as u64,
extent.parity_block_count as u64,
"encrypted object shard count overflow",
)?;
if total > 65_535 {
return Err(FormatError::FecTooManyShards(total as usize));
}
let expected = checked_u64_mul(
extent.data_block_count as u64,
crypto_header.block_size as u64,
"encrypted object size overflow",
)?;
if expected != extent.encrypted_size as u64 {
return Err(FormatError::InvalidArchive(
"encrypted object size is not data_block_count * block_size",
));
}
if extent.encrypted_size as usize <= crypto_header.aead_algo.tag_len() {
return Err(FormatError::InvalidArchive(
"encrypted object is too small for AEAD tag",
));
}
Ok(())
}
fn required_object_parity(
data_block_count: u64,
crypto_header: &CryptoHeaderFixed,
) -> Result<u32, FormatError> {
let min_parity =
if crypto_header.volume_loss_tolerance > 0 || crypto_header.bit_rot_buffer_pct > 0 {
1
} else {
0
};
let mut parity = 0u64;
for _ in 0..100 {
let total = data_block_count
.checked_add(parity)
.ok_or(FormatError::InvalidArchive("parity total overflow"))?;
let by_volume = checked_u64_mul(
crypto_header.volume_loss_tolerance as u64,
ceil_div_u64(total, crypto_header.stripe_width as u64)?,
"volume-loss parity overflow",
)?;
let by_bitrot = ceil_div_u64(
checked_u64_mul(
total,
crypto_header.bit_rot_buffer_pct as u64,
"bit-rot parity overflow",
)?,
100,
)?;
let next = by_volume
.checked_add(by_bitrot)
.ok_or(FormatError::InvalidArchive("parity overflow"))?
.max(min_parity);
if next == parity {
return u32::try_from(next)
.map_err(|_| FormatError::InvalidArchive("parity count overflow"));
}
parity = next;
}
Err(FormatError::InvalidArchive(
"parity calculation did not converge",
))
}
fn ceil_div_u64(numerator: u64, denominator: u64) -> Result<u64, FormatError> {
if denominator == 0 {
return Err(FormatError::InvalidArchive("division by zero"));
}
numerator
.checked_add(denominator - 1)
.ok_or(FormatError::InvalidArchive("ceiling division overflow"))
.map(|value| value / denominator)
}
fn frame_range_for_file<'b>(
shard: &'b IndexShard,
file: &FileEntry,
) -> Result<Vec<&'b FrameEntry>, FormatError> {
let mut frames = Vec::with_capacity(file.frame_count as usize);
for offset in 0..file.frame_count as u64 {
let frame_index =
file.first_frame_index
.checked_add(offset)
.ok_or(FormatError::InvalidArchive(
"FileEntry frame range overflow",
))?;
let frame = shard
.frames
.iter()
.find(|entry| entry.frame_index == frame_index)
.ok_or(FormatError::InvalidArchive(
"FileEntry references missing FrameEntry",
))?;
frames.push(frame);
}
Ok(frames)
}
fn metadata_limits(crypto_header: &CryptoHeaderFixed) -> MetadataLimits {
MetadataLimits {
block_size: crypto_header.block_size,
max_path_length: crypto_header.max_path_length,
max_payload_data_shards: crypto_header.fec_data_shards,
max_payload_parity_shards: crypto_header.fec_parity_shards,
max_index_data_shards: crypto_header.index_fec_data_shards,
max_index_parity_shards: crypto_header.index_fec_parity_shards,
max_index_root_data_shards: crypto_header.index_root_fec_data_shards,
max_index_root_parity_shards: crypto_header.index_root_fec_parity_shards,
..MetadataLimits::default()
}
}
fn verify_dense_keys<T>(
entries: &BTreeMap<u64, T>,
expected_count: u64,
structure: &'static str,
) -> Result<(), FormatError> {
if entries.len() as u64 != expected_count {
return Err(FormatError::InvalidArchive(
"decoded table count does not match IndexRoot",
));
}
for expected in 0..expected_count {
if !entries.contains_key(&expected) {
return Err(FormatError::InvalidMetadata {
structure,
reason: "global index coverage has a gap",
});
}
}
Ok(())
}
fn validate_envelope_frame_coverage(
frames: &BTreeMap<u64, FrameEntry>,
envelopes: &BTreeMap<u64, EnvelopeEntry>,
) -> Result<(), FormatError> {
let mut accounted_frames = BTreeSet::new();
for envelope in envelopes.values() {
let first = envelope.first_frame_index;
let end =
first
.checked_add(envelope.frame_count as u64)
.ok_or(FormatError::InvalidArchive(
"EnvelopeEntry frame range overflow",
))?;
let mut ranges = Vec::with_capacity(envelope.frame_count as usize);
for frame_index in first..end {
let frame = frames.get(&frame_index).ok_or(FormatError::InvalidArchive(
"EnvelopeEntry references missing FrameEntry",
))?;
if frame.envelope_index != envelope.envelope_index {
return Err(FormatError::InvalidArchive(
"FrameEntry envelope_index does not match containing EnvelopeEntry",
));
}
if !accounted_frames.insert(frame_index) {
return Err(FormatError::InvalidArchive(
"FrameEntry is covered by multiple EnvelopeEntries",
));
}
let start = frame.offset_in_envelope as usize;
let end = checked_add(start, frame.compressed_size as usize, "FrameEntry")?;
if end > envelope.plaintext_size as usize {
return Err(FormatError::InvalidArchive(
"FrameEntry exceeds EnvelopeEntry plaintext_size",
));
}
ranges.push((start, end));
}
validate_exact_coverage_ranges(
&mut ranges,
envelope.plaintext_size as usize,
"EnvelopeEntry frame coverage has a gap or overlap",
)?;
}
for frame_index in frames.keys() {
if !accounted_frames.contains(frame_index) {
return Err(FormatError::InvalidArchive(
"FrameEntry is not covered by any EnvelopeEntry",
));
}
}
Ok(())
}
fn validate_file_extent_coverage_ranges(
extents: &[(u64, u64)],
tar_len: usize,
) -> Result<(), FormatError> {
let mut ranges = Vec::with_capacity(extents.len());
for (start, len) in extents {
let start = to_usize(*start, "FileEntry")?;
let len = to_usize(*len, "FileEntry")?;
let end = checked_add(start, len, "FileEntry")?;
if end > tar_len {
return Err(FormatError::InvalidArchive(
"FileEntry extent exceeds IndexRoot tar_total_size",
));
}
ranges.push((start, end));
}
validate_exact_coverage_ranges(
&mut ranges,
tar_len,
"FileEntry extents do not cover tar stream exactly",
)
}
fn add_expected_directory_hint_rows(
map: &mut DirectoryHintMap,
shard_row_index: u32,
path: &[u8],
kind: TarEntryKind,
) {
map.entry(Vec::new()).or_default().insert(shard_row_index);
for (idx, byte) in path.iter().enumerate() {
if *byte == b'/' {
map.entry(path[..idx].to_vec())
.or_default()
.insert(shard_row_index);
}
}
if kind == TarEntryKind::Directory {
map.entry(path.to_vec())
.or_default()
.insert(shard_row_index);
}
}
fn validate_directory_hint_tables_against_expected(
tables: &[DirectoryHintTable],
expected: &DirectoryHintMap,
) -> Result<(), FormatError> {
let mut actual = Vec::new();
let mut previous_key: Option<([u8; 8], Vec<u8>)> = None;
for table in tables {
for entry_index in 0..table.entries.len() {
let path = table
.entry_path(entry_index)
.ok_or(FormatError::InvalidArchive(
"DirectoryHintEntry path is missing",
))?;
let key = (hash_prefix(path), path.to_vec());
if let Some(previous) = &previous_key {
if previous >= &key {
return Err(FormatError::InvalidArchive(
"DirectoryHintEntry rows are not globally sorted",
));
}
}
previous_key = Some(key);
let rows =
table
.shard_rows_for_entry(entry_index)
.ok_or(FormatError::InvalidArchive(
"DirectoryHintEntry shard rows are missing",
))?;
actual.push((path.to_vec(), rows.to_vec()));
}
}
if actual != sorted_directory_hint_rows(expected) {
return Err(FormatError::InvalidArchive(
"directory hint map does not match decoded files",
));
}
Ok(())
}
fn sorted_directory_hint_rows(map: &DirectoryHintMap) -> Vec<(Vec<u8>, Vec<u32>)> {
let mut rows = map
.iter()
.map(|(path, shard_rows)| {
(
path.clone(),
shard_rows.iter().copied().collect::<Vec<u32>>(),
)
})
.collect::<Vec<_>>();
rows.sort_by(|(left_path, _), (right_path, _)| {
hash_prefix(left_path)
.cmp(&hash_prefix(right_path))
.then_with(|| left_path.cmp(right_path))
});
rows
}
fn validate_exact_coverage_ranges(
ranges: &mut [(usize, usize)],
expected_end: usize,
reason: &'static str,
) -> Result<(), FormatError> {
ranges.sort_unstable();
let mut cursor = 0usize;
for (start, end) in ranges.iter().copied() {
if start != cursor || end < start {
return Err(FormatError::InvalidArchive(reason));
}
cursor = end;
}
if cursor != expected_end {
return Err(FormatError::InvalidArchive(reason));
}
Ok(())
}
fn object_block_range(
first_block_index: u64,
data_block_count: u32,
parity_block_count: u32,
structure: &'static str,
) -> Result<(u64, u64), FormatError> {
let total = data_block_count as u64 + parity_block_count as u64;
if total == 0 {
return Err(FormatError::InvalidArchive(structure));
}
let end = checked_u64_add(first_block_index, total, structure)?;
Ok((first_block_index, end))
}
fn validate_non_overlapping_object_ranges(ranges: &mut [(u64, u64)]) -> Result<(), FormatError> {
ranges.sort_unstable();
for pair in ranges.windows(2) {
if pair[0].1 > pair[1].0 {
return Err(FormatError::InvalidArchive(
"encrypted object block ranges overlap",
));
}
}
Ok(())
}
fn observed_archive_size(sizes: impl IntoIterator<Item = u64>) -> Result<u64, FormatError> {
sizes.into_iter().try_fold(0u64, |sum, size| {
sum.checked_add(size).ok_or(FormatError::InvalidArchive(
"observed archive size overflow",
))
})
}
fn total_extraction_size_cap(options: ReaderOptions, observed_archive_bytes: u64) -> u64 {
options
.max_total_extraction_size
.min(observed_archive_bytes.saturating_mul(10))
}
fn utf8_path(bytes: &[u8]) -> Result<String, FormatError> {
std::str::from_utf8(bytes)
.map(|path| path.to_owned())
.map_err(|_| FormatError::UnsafeArchivePath)
}
fn sha256_bytes(bytes: &[u8]) -> [u8; 32] {
let digest = Sha256::digest(bytes);
let mut out = [0u8; 32];
out.copy_from_slice(&digest);
out
}
fn slice<'b>(
bytes: &'b [u8],
offset: usize,
len: usize,
structure: &'static str,
) -> Result<&'b [u8], FormatError> {
let end = checked_add(offset, len, structure)?;
bytes.get(offset..end).ok_or(FormatError::InvalidLength {
structure,
expected: end,
actual: bytes.len(),
})
}
fn checked_add(lhs: usize, rhs: usize, structure: &'static str) -> Result<usize, FormatError> {
lhs.checked_add(rhs)
.ok_or(FormatError::InvalidArchive(structure))
}
fn checked_u64_add(lhs: u64, rhs: u64, structure: &'static str) -> Result<u64, FormatError> {
lhs.checked_add(rhs)
.ok_or(FormatError::InvalidArchive(structure))
}
fn to_usize(value: u64, structure: &'static str) -> Result<usize, FormatError> {
usize::try_from(value).map_err(|_| FormatError::InvalidArchive(structure))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::crypto::compute_hmac;
use crate::format::{AeadAlgo, CompressionAlgo, FecAlgo, KdfAlgo};
use crate::metadata::{DirectoryHintEntry, DirectoryHintTableHeader};
use crate::writer::{write_archive, write_archive_with_dictionary, RegularFile, WriterOptions};
fn master_key() -> MasterKey {
MasterKey::from_raw_key(&[0x42; 32]).unwrap()
}
fn dictionary() -> &'static [u8] {
b"dir/dict.txt common words common words common words dictionary payload"
}
fn single_stream_options() -> WriterOptions {
WriterOptions {
stripe_width: 1,
volume_loss_tolerance: 0,
..WriterOptions::default()
}
}
#[test]
fn opens_lists_verifies_and_extracts_one_file_archive() {
let archive = write_archive(
&[RegularFile::new("dir/hello.txt", b"hello m7")],
&master_key(),
single_stream_options(),
)
.unwrap();
let opened = open_archive(&archive.bytes, &master_key()).unwrap();
assert_eq!(
opened.list_files().unwrap(),
vec![ArchiveEntry {
path: "dir/hello.txt".to_string(),
file_data_size: 8,
kind: TarEntryKind::Regular,
mode: 0o644,
mtime: 0,
diagnostics: Vec::new(),
}]
);
opened.verify().unwrap();
assert_eq!(
opened.extract_file("dir/hello.txt").unwrap(),
Some(b"hello m7".to_vec())
);
assert_eq!(opened.extract_file("missing.txt").unwrap(), None);
}
#[test]
fn safe_extract_writes_regular_file_under_root() {
let archive = write_archive(
&[RegularFile::new("dir/hello.txt", b"safe m8")],
&master_key(),
single_stream_options(),
)
.unwrap();
let opened = open_archive(&archive.bytes, &master_key()).unwrap();
let tmp = tempfile::tempdir().unwrap();
opened
.extract_file_to(
"dir/hello.txt",
tmp.path(),
SafeExtractionOptions::default(),
)
.unwrap()
.unwrap();
assert_eq!(
std::fs::read(tmp.path().join("dir").join("hello.txt")).unwrap(),
b"safe m8"
);
}
#[test]
fn safe_extract_rejects_overwriting_existing_file_by_default() {
let archive = write_archive(
&[RegularFile::new("hello.txt", b"new")],
&master_key(),
single_stream_options(),
)
.unwrap();
let opened = open_archive(&archive.bytes, &master_key()).unwrap();
let tmp = tempfile::tempdir().unwrap();
std::fs::write(tmp.path().join("hello.txt"), b"old").unwrap();
assert_eq!(
opened
.extract_file_to("hello.txt", tmp.path(), SafeExtractionOptions::default())
.unwrap_err(),
FormatError::UnsafeOverwrite
);
assert_eq!(std::fs::read(tmp.path().join("hello.txt")).unwrap(), b"old");
}
#[test]
fn opens_and_verifies_empty_archive() {
let archive = write_archive(&[], &master_key(), single_stream_options()).unwrap();
let opened = open_archive(&archive.bytes, &master_key()).unwrap();
assert!(opened.list_files().unwrap().is_empty());
opened.verify().unwrap();
}
#[test]
fn default_reader_options_allow_v36_trailing_garbage_scan() {
let archive = write_archive(
&[RegularFile::new("garbage-tolerant.txt", b"still intact")],
&master_key(),
single_stream_options(),
)
.unwrap();
let mut with_trailing_garbage = archive.bytes.clone();
with_trailing_garbage.extend_from_slice(b"ignored trailing bytes");
let opened = open_archive(&with_trailing_garbage, &master_key()).unwrap();
assert_eq!(
opened.extract_file("garbage-tolerant.txt").unwrap(),
Some(b"still intact".to_vec())
);
}
#[test]
fn rejects_wrong_key_before_metadata_release() {
let archive = write_archive(&[], &master_key(), single_stream_options()).unwrap();
let wrong = MasterKey::from_raw_key(&[0x43; 32]).unwrap();
assert_eq!(
open_archive(&archive.bytes, &wrong).unwrap_err(),
FormatError::HmacMismatch {
structure: "CryptoHeader"
}
);
}
#[test]
fn rejects_payload_tamper_even_with_recomputed_block_crc() {
let mut archive = write_archive(
&[RegularFile::new("file.txt", b"authenticated")],
&master_key(),
single_stream_options(),
)
.unwrap()
.bytes;
let volume = VolumeHeader::parse(&archive[..VOLUME_HEADER_LEN]).unwrap();
let crypto_end = VOLUME_HEADER_LEN + usize::try_from(volume.crypto_header_length).unwrap();
let crypto = CryptoHeader::parse(
&archive[VOLUME_HEADER_LEN..crypto_end],
volume.crypto_header_length,
)
.unwrap();
let block_size = crypto.fixed.block_size as usize;
archive[crypto_end + 16] ^= 1;
let crc_offset = crypto_end + 16 + block_size;
let crc = crc32c::crc32c(&archive[crypto_end..crc_offset]);
archive[crc_offset..crc_offset + 4].copy_from_slice(&crc.to_le_bytes());
let opened = open_archive(&archive, &master_key()).unwrap();
assert_eq!(opened.verify().unwrap_err(), FormatError::AeadFailure);
}
#[test]
fn list_and_extract_use_final_view_for_duplicate_paths() {
let archive = write_archive(
&[
RegularFile::new("same.txt", b"old"),
RegularFile::new("same.txt", b"newer"),
],
&master_key(),
single_stream_options(),
)
.unwrap();
let opened = open_archive(&archive.bytes, &master_key()).unwrap();
assert_eq!(
opened.list_files().unwrap(),
vec![ArchiveEntry {
path: "same.txt".to_string(),
file_data_size: 5,
kind: TarEntryKind::Regular,
mode: 0o644,
mtime: 0,
diagnostics: Vec::new(),
}]
);
assert_eq!(
opened.extract_file("same.txt").unwrap(),
Some(b"newer".to_vec())
);
opened.verify().unwrap();
}
#[test]
fn bootstrap_sidecar_opens_lists_verifies_and_extracts() {
let archive = write_archive(
&[RegularFile::new("dir/sidecar.txt", b"hello sidecar")],
&master_key(),
single_stream_options(),
)
.unwrap();
let opened = open_archive_with_bootstrap_sidecar(
&archive.bytes,
&archive.bootstrap_sidecar,
&master_key(),
)
.unwrap();
assert_eq!(
opened.list_files().unwrap(),
vec![ArchiveEntry {
path: "dir/sidecar.txt".to_string(),
file_data_size: 13,
kind: TarEntryKind::Regular,
mode: 0o644,
mtime: 0,
diagnostics: Vec::new(),
}]
);
assert_eq!(
opened.extract_file("dir/sidecar.txt").unwrap(),
Some(b"hello sidecar".to_vec())
);
opened.verify().unwrap();
}
#[test]
fn dictionary_archive_opens_lists_verifies_and_extracts_seekable() {
let archive = write_archive_with_dictionary(
&[RegularFile::new(
"dir/dict.txt",
b"common words common words dictionary payload",
)],
&master_key(),
single_stream_options(),
dictionary(),
)
.unwrap();
let opened = open_archive(&archive.bytes, &master_key()).unwrap();
assert_eq!(opened.crypto_header.has_dictionary, 1);
assert!(opened.index_root.header.dictionary_data_block_count > 0);
assert_eq!(
opened.list_files().unwrap(),
vec![ArchiveEntry {
path: "dir/dict.txt".to_string(),
file_data_size: 44,
kind: TarEntryKind::Regular,
mode: 0o644,
mtime: 0,
diagnostics: Vec::new(),
}]
);
assert_eq!(
opened.extract_file("dir/dict.txt").unwrap(),
Some(b"common words common words dictionary payload".to_vec())
);
opened.verify().unwrap();
}
#[test]
fn dictionary_object_tamper_fails_before_payload_decompression() {
let archive = write_archive_with_dictionary(
&[RegularFile::new(
"dir/dict.txt",
b"common words common words dictionary payload",
)],
&master_key(),
single_stream_options(),
dictionary(),
)
.unwrap();
let opened = open_archive(&archive.bytes, &master_key()).unwrap();
let volume_header = VolumeHeader::parse(&archive.bytes[..VOLUME_HEADER_LEN]).unwrap();
let crypto_end = VOLUME_HEADER_LEN + volume_header.crypto_header_length as usize;
let record_len = opened.crypto_header.block_size as usize + BLOCK_RECORD_FRAMING_LEN;
let dictionary_offset =
crypto_end + opened.index_root.header.dictionary_first_block as usize * record_len;
let mut tampered = archive.bytes.clone();
tampered[dictionary_offset + 16] ^= 0x01;
let crc_offset = dictionary_offset + 16 + opened.crypto_header.block_size as usize;
let crc = crc32c::crc32c(&tampered[dictionary_offset..crc_offset]);
tampered[crc_offset..crc_offset + 4].copy_from_slice(&crc.to_le_bytes());
assert_eq!(
open_archive(&tampered, &master_key()).unwrap_err(),
FormatError::AeadFailure
);
}
#[test]
fn dictionary_archive_bootstraps_from_sidecar_for_non_seekable_open() {
let archive = write_archive_with_dictionary(
&[RegularFile::new(
"dict-sidecar.txt",
b"common words common words sidecar payload",
)],
&master_key(),
single_stream_options(),
dictionary(),
)
.unwrap();
let opened = open_non_seekable_archive(
&archive.bytes,
&master_key(),
Some(&archive.bootstrap_sidecar),
)
.unwrap();
assert_eq!(
opened.extract_file("dict-sidecar.txt").unwrap(),
Some(b"common words common words sidecar payload".to_vec())
);
opened.verify().unwrap();
}
#[test]
fn bootstrap_sidecar_treats_crc_failed_payload_block_as_erasure() {
let archive = write_archive(
&[RegularFile::new(
"sidecar-erasure.txt",
b"repair through sidecar",
)],
&master_key(),
single_stream_options(),
)
.unwrap();
let mut corrupted = archive.bytes.clone();
corrupt_first_block_record_payload(&mut corrupted);
let opened = open_archive_with_bootstrap_sidecar(
&corrupted,
&archive.bootstrap_sidecar,
&master_key(),
)
.unwrap();
assert_eq!(
opened.extract_file("sidecar-erasure.txt").unwrap(),
Some(b"repair through sidecar".to_vec())
);
}
#[test]
fn extraction_rejects_logical_payload_above_total_size_cap() {
let archive = write_archive(
&[RegularFile::new("cap.txt", b"payload")],
&master_key(),
single_stream_options(),
)
.unwrap();
let mut options = ReaderOptions::default();
options.max_total_extraction_size = 3;
let opened =
OpenedArchive::open_with_options(&archive.bytes, &master_key(), options).unwrap();
assert_eq!(
opened.extract_file("cap.txt").unwrap_err(),
FormatError::ReaderUnsupported("total extraction size exceeds configured cap")
);
}
#[test]
fn verify_does_not_apply_extraction_payload_cap() {
let archive = write_archive(
&[RegularFile::new("verify-cap.txt", b"payload")],
&master_key(),
single_stream_options(),
)
.unwrap();
let mut options = ReaderOptions::default();
options.max_total_extraction_size = 3;
let opened =
OpenedArchive::open_with_options(&archive.bytes, &master_key(), options).unwrap();
opened.verify().unwrap();
assert_eq!(
opened.extract_file("verify-cap.txt").unwrap_err(),
FormatError::ReaderUnsupported("total extraction size exceeds configured cap")
);
}
#[test]
fn dictionary_sidecar_requires_dictionary_record_section() {
let archive = write_archive_with_dictionary(
&[RegularFile::new("dict-missing.txt", b"common words")],
&master_key(),
single_stream_options(),
dictionary(),
)
.unwrap();
let header = BootstrapSidecarHeader::parse(
&archive.bootstrap_sidecar[..BOOTSTRAP_SIDECAR_HEADER_LEN],
)
.unwrap();
let mut missing_dictionary =
archive.bootstrap_sidecar[..header.dictionary_records_offset as usize].to_vec();
rewrite_sidecar_header(&mut missing_dictionary, &master_key(), |header| {
header.flags &= !0x04;
header.dictionary_records_offset = 0;
header.dictionary_records_length = 0;
});
assert_eq!(
open_archive_with_bootstrap_sidecar(&archive.bytes, &missing_dictionary, &master_key())
.unwrap_err(),
FormatError::ReaderUnsupported("dictionary bootstrap required")
);
}
#[test]
fn dictionary_sidecar_records_are_validated_against_dictionary_extent() {
let archive = write_archive_with_dictionary(
&[RegularFile::new("dict-sidecar-kind.txt", b"common words")],
&master_key(),
single_stream_options(),
dictionary(),
)
.unwrap();
let mut wrong_kind = archive.bootstrap_sidecar.clone();
mutate_sidecar_dictionary_record(&mut wrong_kind, 0, |record| {
record.kind = BlockKind::IndexRootData;
});
assert_eq!(
open_archive_with_bootstrap_sidecar(&archive.bytes, &wrong_kind, &master_key())
.unwrap_err(),
FormatError::InvalidArchive("sidecar BlockRecord section has wrong kind")
);
let mut wrong_last = archive.bootstrap_sidecar.clone();
mutate_sidecar_dictionary_record(&mut wrong_last, 0, |record| {
record.flags = 0;
});
assert_eq!(
open_archive_with_bootstrap_sidecar(&archive.bytes, &wrong_last, &master_key())
.unwrap_err(),
FormatError::InvalidArchive("sidecar BlockRecord section has wrong last-data flag")
);
}
#[test]
fn non_seekable_random_access_requires_sidecar() {
let archive = write_archive(
&[RegularFile::new("file.txt", b"payload")],
&master_key(),
single_stream_options(),
)
.unwrap();
assert_eq!(
open_non_seekable_archive(&archive.bytes, &master_key(), None).unwrap_err(),
FormatError::ReaderUnsupported(
"non-seekable random access requires a bootstrap sidecar"
)
);
assert!(open_non_seekable_archive(
&archive.bytes,
&master_key(),
Some(&archive.bootstrap_sidecar)
)
.is_ok());
}
#[test]
fn sequential_extracts_dictionary_free_tar_stream() {
let archive = write_archive(
&[RegularFile::new("seq.txt", b"streaming")],
&master_key(),
single_stream_options(),
)
.unwrap();
let tar_stream = sequential_extract_tar_stream(&archive.bytes, &master_key()).unwrap();
let member = parse_tar_member_group(&tar_stream, 4096).unwrap();
assert_eq!(member.path, b"seq.txt");
assert_eq!(member.data, b"streaming");
}
#[test]
fn sequential_rejects_logical_payload_above_total_size_cap() {
let archive = write_archive(
&[RegularFile::new("seq-cap.txt", b"payload")],
&master_key(),
single_stream_options(),
)
.unwrap();
let mut options = ReaderOptions::default();
options.max_total_extraction_size = 3;
assert_eq!(
sequential_extract_tar_stream_with_options(&archive.bytes, &master_key(), options)
.unwrap_err(),
FormatError::ReaderUnsupported("total extraction size exceeds configured cap")
);
}
#[test]
fn sequential_repairs_crc_failed_payload_data_when_parity_is_guaranteed() {
let archive = write_archive(
&[RegularFile::new("seq-erasure.txt", b"stream repair")],
&master_key(),
single_stream_options(),
)
.unwrap();
let mut corrupted = archive.bytes;
corrupt_first_block_record_payload(&mut corrupted);
let tar_stream = sequential_extract_tar_stream(&corrupted, &master_key()).unwrap();
let member = parse_tar_member_group(&tar_stream, 4096).unwrap();
assert_eq!(member.path, b"seq-erasure.txt");
assert_eq!(member.data, b"stream repair");
}
#[test]
fn sequential_rejects_crc_failed_payload_data_without_guaranteed_parity() {
let archive = write_archive(
&[RegularFile::new("seq-no-parity.txt", b"no repair")],
&master_key(),
WriterOptions {
bit_rot_buffer_pct: 0,
fec_parity_shards: 0,
index_fec_parity_shards: 0,
index_root_fec_parity_shards: 0,
..single_stream_options()
},
)
.unwrap();
let mut corrupted = archive.bytes;
corrupt_first_block_record_payload(&mut corrupted);
assert_eq!(
sequential_extract_tar_stream(&corrupted, &master_key()).unwrap_err(),
FormatError::BadCrc {
structure: "BlockRecord"
}
);
}
#[test]
fn sequential_rejects_when_terminal_authentication_fails() {
let archive = write_archive(
&[RegularFile::new("seq.txt", b"streaming")],
&master_key(),
single_stream_options(),
)
.unwrap();
let mut corrupted = archive.bytes;
let trailer_hmac_offset = corrupted.len() - VOLUME_TRAILER_LEN + TRAILER_HMAC_COVERED_LEN;
corrupted[trailer_hmac_offset] ^= 0x01;
assert_eq!(
sequential_extract_tar_stream(&corrupted, &master_key()).unwrap_err(),
FormatError::HmacMismatch {
structure: "VolumeTrailer"
}
);
}
#[test]
fn sequential_zstd_stream_rejects_skippable_frame_segments() {
let skippable = [0x50, 0x2a, 0x4d, 0x18, 0, 0, 0, 0];
let mut output = Vec::new();
assert_eq!(
decode_concatenated_zstd_frames(&skippable, None, &mut output).unwrap_err(),
FormatError::NotStandardZstdFrame
);
assert!(output.is_empty());
}
#[test]
fn bootstrap_sidecar_rejects_bad_flags_and_trailing_bytes() {
let archive = write_archive(&[], &master_key(), single_stream_options()).unwrap();
let mut bad_flags = archive.bootstrap_sidecar.clone();
rewrite_sidecar_header(&mut bad_flags, &master_key(), |header| {
header.flags |= 0x08;
});
assert_eq!(
open_archive_with_bootstrap_sidecar(&archive.bytes, &bad_flags, &master_key())
.unwrap_err(),
FormatError::UnknownBootstrapSidecarFlags(0x0b)
);
let mut trailing = archive.bootstrap_sidecar.clone();
trailing.push(0);
assert_eq!(
open_archive_with_bootstrap_sidecar(&archive.bytes, &trailing, &master_key())
.unwrap_err(),
FormatError::NonCanonicalBootstrapSidecarLayout
);
}
#[test]
fn bootstrap_sidecar_rejects_bad_manifest_footer_semantics() {
let archive = write_archive(&[], &master_key(), single_stream_options()).unwrap();
let mut wrong_volume = archive.bootstrap_sidecar.clone();
mutate_sidecar_manifest(&mut wrong_volume, &master_key(), |footer| {
footer.volume_index = 1;
});
assert_eq!(
open_archive_with_bootstrap_sidecar(&archive.bytes, &wrong_volume, &master_key())
.unwrap_err(),
FormatError::InvalidArchive("sidecar ManifestFooter volume_index must be zero")
);
let mut non_authoritative = archive.bootstrap_sidecar.clone();
mutate_sidecar_manifest(&mut non_authoritative, &master_key(), |footer| {
footer.is_authoritative = 0;
});
assert_eq!(
open_archive_with_bootstrap_sidecar(&archive.bytes, &non_authoritative, &master_key())
.unwrap_err(),
FormatError::InvalidArchive("sidecar ManifestFooter is not authoritative")
);
}
#[test]
fn bootstrap_sidecar_rejects_dictionary_section_for_no_dictionary_archive() {
let archive = write_archive(&[], &master_key(), single_stream_options()).unwrap();
let mut with_dictionary = archive.bootstrap_sidecar.clone();
let header =
BootstrapSidecarHeader::parse(&with_dictionary[..BOOTSTRAP_SIDECAR_HEADER_LEN])
.unwrap();
let record_len = sidecar_record_len(&with_dictionary);
let first_record = header.index_root_records_offset as usize;
let copied_record = with_dictionary[first_record..first_record + record_len].to_vec();
let dictionary_offset = with_dictionary.len() as u64;
with_dictionary.extend_from_slice(&copied_record);
rewrite_sidecar_header(&mut with_dictionary, &master_key(), |header| {
header.flags |= 0x04;
header.dictionary_records_offset = dictionary_offset;
header.dictionary_records_length = record_len as u64;
});
assert_eq!(
open_archive_with_bootstrap_sidecar(&archive.bytes, &with_dictionary, &master_key())
.unwrap_err(),
FormatError::InvalidArchive(
"bootstrap sidecar has dictionary records while has_dictionary is false"
)
);
}
#[test]
fn bootstrap_sidecar_rejects_missing_duplicate_wrong_kind_and_wrong_last_flag() {
let archive = write_archive(&[], &master_key(), single_stream_options()).unwrap();
let mut missing = archive.bootstrap_sidecar.clone();
let record_len = sidecar_record_len(&missing);
let new_len = missing.len() - record_len;
missing.truncate(new_len);
rewrite_sidecar_header(&mut missing, &master_key(), |header| {
header.index_root_records_length -= record_len as u64;
});
assert_eq!(
open_archive_with_bootstrap_sidecar(&archive.bytes, &missing, &master_key())
.unwrap_err(),
FormatError::InvalidArchive(
"sidecar BlockRecord section does not match declared extent"
)
);
let mut duplicate = archive.bootstrap_sidecar.clone();
mutate_sidecar_index_record(&mut duplicate, 1, |record| {
record.block_index -= 1;
});
assert_eq!(
open_archive_with_bootstrap_sidecar(&archive.bytes, &duplicate, &master_key())
.unwrap_err(),
FormatError::InvalidArchive(
"sidecar BlockRecord section has missing or duplicate blocks"
)
);
let mut misordered = archive.bootstrap_sidecar.clone();
swap_sidecar_index_records(&mut misordered, 0, 1);
assert_eq!(
open_archive_with_bootstrap_sidecar(&archive.bytes, &misordered, &master_key())
.unwrap_err(),
FormatError::InvalidArchive(
"sidecar BlockRecord section has missing or duplicate blocks"
)
);
let mut wrong_kind = archive.bootstrap_sidecar.clone();
mutate_sidecar_index_record(&mut wrong_kind, 0, |record| {
record.kind = BlockKind::PayloadData;
});
assert_eq!(
open_archive_with_bootstrap_sidecar(&archive.bytes, &wrong_kind, &master_key())
.unwrap_err(),
FormatError::InvalidArchive("sidecar BlockRecord section has wrong kind")
);
let mut wrong_last = archive.bootstrap_sidecar.clone();
mutate_sidecar_index_record(&mut wrong_last, 0, |record| {
record.flags = 0;
});
assert_eq!(
open_archive_with_bootstrap_sidecar(&archive.bytes, &wrong_last, &master_key())
.unwrap_err(),
FormatError::InvalidArchive("sidecar BlockRecord section has wrong last-data flag")
);
}
#[test]
fn verify_helper_rejects_envelope_frame_coverage_gap() {
let frames = BTreeMap::from([(
0,
FrameEntry {
frame_index: 0,
envelope_index: 0,
offset_in_envelope: 0,
compressed_size: 10,
decompressed_size: 512,
flags: 0,
tar_stream_offset: 0,
},
)]);
let envelopes = BTreeMap::from([(
0,
EnvelopeEntry {
envelope_index: 0,
first_block_index: 0,
data_block_count: 1,
parity_block_count: 1,
encrypted_size: 4096,
plaintext_size: 11,
first_frame_index: 0,
frame_count: 1,
},
)]);
assert_eq!(
validate_envelope_frame_coverage(&frames, &envelopes).unwrap_err(),
FormatError::InvalidArchive("EnvelopeEntry frame coverage has a gap or overlap")
);
}
#[test]
fn verify_helper_rejects_file_extent_gaps_and_overlaps() {
assert!(validate_file_extent_coverage_ranges(&[(512, 512), (0, 512)], 1024).is_ok());
assert_eq!(
validate_file_extent_coverage_ranges(&[(0, 512), (1024, 512)], 1536).unwrap_err(),
FormatError::InvalidArchive("FileEntry extents do not cover tar stream exactly")
);
assert_eq!(
validate_file_extent_coverage_ranges(&[(0, 1024), (512, 512)], 1024).unwrap_err(),
FormatError::InvalidArchive("FileEntry extents do not cover tar stream exactly")
);
}
#[test]
fn expected_directory_hint_rows_include_ancestors_and_directory_entries() {
let mut map = DirectoryHintMap::new();
add_expected_directory_hint_rows(&mut map, 2, b"foo/bar/baz.txt", TarEntryKind::Regular);
add_expected_directory_hint_rows(&mut map, 4, b"foo/bar", TarEntryKind::Directory);
assert_eq!(map.get(&Vec::new()), Some(&BTreeSet::from([2, 4])));
assert_eq!(map.get(&b"foo".to_vec()), Some(&BTreeSet::from([2, 4])));
assert_eq!(map.get(&b"foo/bar".to_vec()), Some(&BTreeSet::from([2, 4])));
assert!(!map.contains_key(&b"foo/bar/baz.txt".to_vec()));
assert!(!map.contains_key(&b"foobar".to_vec()));
}
#[test]
fn directory_hint_validation_requires_exact_global_map() {
let mut expected = DirectoryHintMap::new();
add_expected_directory_hint_rows(&mut expected, 0, b"foo/bar.txt", TarEntryKind::Regular);
add_expected_directory_hint_rows(&mut expected, 1, b"foo", TarEntryKind::Directory);
let rows = sorted_directory_hint_rows(&expected);
let table = directory_hint_table_from_rows(7, &rows, 2);
validate_directory_hint_tables_against_expected(&[table.clone()], &expected).unwrap();
let mut incomplete = expected.clone();
incomplete.get_mut(&b"foo".to_vec()).unwrap().remove(&1);
assert_eq!(
validate_directory_hint_tables_against_expected(&[table], &incomplete).unwrap_err(),
FormatError::InvalidArchive("directory hint map does not match decoded files")
);
}
#[test]
fn directory_hint_validation_rejects_global_order_mismatch() {
let mut expected = DirectoryHintMap::new();
expected.insert(Vec::new(), BTreeSet::from([0]));
expected.insert(b"alpha".to_vec(), BTreeSet::from([0]));
let rows = sorted_directory_hint_rows(&expected);
let first = directory_hint_table_from_rows(8, &rows[..1], 1);
let second = directory_hint_table_from_rows(9, &rows[1..], 1);
assert_eq!(
validate_directory_hint_tables_against_expected(&[second, first], &expected)
.unwrap_err(),
FormatError::InvalidArchive("DirectoryHintEntry rows are not globally sorted")
);
}
#[test]
fn object_extent_rejects_parity_above_class_cap() {
let crypto_header = CryptoHeaderFixed {
length: 0,
compression_algo: CompressionAlgo::ZstdFramed,
aead_algo: AeadAlgo::AesGcmSiv256,
fec_algo: FecAlgo::ReedSolomonGF16,
kdf_algo: KdfAlgo::Raw,
chunk_size: 1024,
envelope_target_size: 4096,
block_size: 4096,
fec_data_shards: 1,
fec_parity_shards: 1,
index_fec_data_shards: 1,
index_fec_parity_shards: 1,
index_root_fec_data_shards: 1,
index_root_fec_parity_shards: 1,
stripe_width: 1,
volume_loss_tolerance: 0,
bit_rot_buffer_pct: 0,
has_dictionary: 0,
max_path_length: 4096,
expected_volume_size: 0,
};
let extent = ObjectExtent {
first_block_index: 0,
data_block_count: 1,
parity_block_count: 2,
encrypted_size: 4096,
};
assert_eq!(
validate_object_extent(extent, &crypto_header, 1, 1).unwrap_err(),
FormatError::InvalidArchive("encrypted object exceeds its class parity-shard maximum")
);
}
#[test]
fn object_extent_rejects_parity_below_recoverability_requirement() {
let crypto_header = CryptoHeaderFixed {
length: 0,
compression_algo: CompressionAlgo::ZstdFramed,
aead_algo: AeadAlgo::AesGcmSiv256,
fec_algo: FecAlgo::ReedSolomonGF16,
kdf_algo: KdfAlgo::Raw,
chunk_size: 1024,
envelope_target_size: 4096,
block_size: 4096,
fec_data_shards: 1,
fec_parity_shards: 1,
index_fec_data_shards: 1,
index_fec_parity_shards: 1,
index_root_fec_data_shards: 1,
index_root_fec_parity_shards: 1,
stripe_width: 2,
volume_loss_tolerance: 1,
bit_rot_buffer_pct: 0,
has_dictionary: 0,
max_path_length: 4096,
expected_volume_size: 0,
};
let extent = ObjectExtent {
first_block_index: 0,
data_block_count: 1,
parity_block_count: 0,
encrypted_size: 4096,
};
assert_eq!(
validate_object_extent(extent, &crypto_header, 1, 1).unwrap_err(),
FormatError::InvalidArchive(
"encrypted object has insufficient parity for recovery settings"
)
);
}
#[test]
fn opens_complete_multi_volume_archive() {
let files = [RegularFile::new("alpha.txt", b"hello from volume stripes")];
let archive = write_archive(
&files,
&master_key(),
WriterOptions {
stripe_width: 2,
volume_loss_tolerance: 1,
..single_stream_options()
},
)
.unwrap();
assert_eq!(archive.volumes.len(), 2);
let volume_refs = archive
.volumes
.iter()
.map(Vec::as_slice)
.collect::<Vec<_>>();
let opened = open_archive_volumes(&volume_refs, &master_key()).unwrap();
assert_eq!(opened.volume_header.stripe_width, 2);
assert_eq!(opened.list_files().unwrap()[0].path, "alpha.txt");
assert_eq!(
opened.extract_file("alpha.txt").unwrap(),
Some(b"hello from volume stripes".to_vec())
);
opened.verify().unwrap();
}
#[test]
fn recovers_from_one_missing_volume_when_parity_allows() {
let files = [RegularFile::new("alpha.txt", b"recover me")];
let archive = write_archive(
&files,
&master_key(),
WriterOptions {
stripe_width: 2,
volume_loss_tolerance: 1,
..single_stream_options()
},
)
.unwrap();
let recovered =
open_archive_volumes(&[archive.volumes[1].as_slice()], &master_key()).unwrap();
assert_eq!(
recovered.extract_file("alpha.txt").unwrap(),
Some(b"recover me".to_vec())
);
recovered.verify().unwrap();
}
#[test]
fn recovers_from_crc_corrupted_block_when_parity_allows() {
let files = [RegularFile::new("alpha.txt", b"repair corrupt block")];
let archive = write_archive(
&files,
&master_key(),
WriterOptions {
stripe_width: 2,
volume_loss_tolerance: 1,
..single_stream_options()
},
)
.unwrap();
let mut volumes = archive.volumes.clone();
corrupt_first_block_record_payload(&mut volumes[0]);
let volume_refs = volumes.iter().map(Vec::as_slice).collect::<Vec<_>>();
let recovered = open_archive_volumes(&volume_refs, &master_key()).unwrap();
assert_eq!(
recovered.extract_file("alpha.txt").unwrap(),
Some(b"repair corrupt block".to_vec())
);
recovered.verify().unwrap();
}
#[test]
fn rejects_block_record_at_wrong_stripe_position() {
let files = [RegularFile::new("alpha.txt", b"wrong stripe")];
let archive = write_archive(
&files,
&master_key(),
WriterOptions {
stripe_width: 2,
volume_loss_tolerance: 1,
..single_stream_options()
},
)
.unwrap();
let mut volumes = archive.volumes.clone();
mutate_first_block_record(&mut volumes[0], |record| {
record.block_index += 2;
});
let volume_refs = volumes.iter().map(Vec::as_slice).collect::<Vec<_>>();
assert_eq!(
open_archive_volumes(&volume_refs, &master_key()).unwrap_err(),
FormatError::InvalidArchive("BlockRecord index does not match volume position")
);
}
#[test]
fn rejects_duplicate_authenticated_volume_indexes() {
let files = [RegularFile::new("alpha.txt", b"duplicates")];
let archive = write_archive(
&files,
&master_key(),
WriterOptions {
stripe_width: 2,
volume_loss_tolerance: 1,
..single_stream_options()
},
)
.unwrap();
assert_eq!(
open_archive_volumes(
&[archive.volumes[0].as_slice(), archive.volumes[0].as_slice()],
&master_key()
)
.unwrap_err(),
FormatError::InvalidArchive("duplicate authenticated volume index")
);
}
fn directory_hint_table_from_rows(
hint_shard_index: u64,
rows: &[(Vec<u8>, Vec<u32>)],
shard_count: u32,
) -> DirectoryHintTable {
let mut entries = Vec::new();
let mut shard_row_indexes = Vec::new();
let mut string_pool = Vec::new();
for (path, rows) in rows {
let path_offset = if path.is_empty() {
0
} else {
let offset = string_pool.len() as u64;
string_pool.extend_from_slice(path);
offset
};
let shard_list_start_index = shard_row_indexes.len() as u32;
shard_row_indexes.extend_from_slice(rows);
entries.push(DirectoryHintEntry {
dir_hash: hash_prefix(path),
path_offset,
path_length: path.len() as u32,
shard_list_start_index,
shard_count: rows.len() as u32,
});
}
let table_bytes =
directory_hint_table_bytes(hint_shard_index, entries, shard_row_indexes, string_pool);
let locating = DirectoryHintShardEntry {
hint_shard_index,
first_dir_hash: hash_prefix(&rows.first().unwrap().0),
last_dir_hash: hash_prefix(&rows.last().unwrap().0),
first_block_index: 0,
data_block_count: 1,
parity_block_count: 0,
encrypted_size: 4096,
decompressed_size: table_bytes.len() as u32,
entry_count: rows.len() as u64,
};
DirectoryHintTable::parse(
&table_bytes,
&locating,
shard_count,
MetadataLimits::default(),
)
.unwrap()
}
fn directory_hint_table_bytes(
hint_shard_index: u64,
entries: Vec<DirectoryHintEntry>,
shard_row_indexes: Vec<u32>,
string_pool: Vec<u8>,
) -> Vec<u8> {
let header_len = DirectoryHintTableHeader {
version: 1,
hint_shard_index,
entry_count: 0,
entry_table_offset: 0,
shard_list_offset: 0,
string_pool_offset: 0,
string_pool_size: 0,
}
.to_bytes()
.len();
let entry_len = entries
.first()
.map(|entry| entry.to_bytes().len())
.unwrap_or(0);
let shard_list_offset = if entries.is_empty() {
0
} else {
header_len + entries.len() * entry_len
};
let string_pool_offset = if string_pool.is_empty() {
0
} else {
shard_list_offset + shard_row_indexes.len() * 4
};
let header = DirectoryHintTableHeader {
version: 1,
hint_shard_index,
entry_count: entries.len() as u64,
entry_table_offset: if entries.is_empty() {
0
} else {
header_len as u64
},
shard_list_offset: shard_list_offset as u64,
string_pool_offset: string_pool_offset as u64,
string_pool_size: string_pool.len() as u64,
};
let mut out = Vec::new();
out.extend_from_slice(&header.to_bytes());
for entry in entries {
out.extend_from_slice(&entry.to_bytes());
}
for row in shard_row_indexes {
out.extend_from_slice(&row.to_le_bytes());
}
out.extend_from_slice(&string_pool);
out
}
fn corrupt_first_block_record_payload(volume: &mut [u8]) {
let (record_offset, _) = first_block_record(volume);
volume[record_offset + 16] ^= 0x55;
}
fn mutate_first_block_record(volume: &mut [u8], mutate: impl FnOnce(&mut BlockRecord)) {
let (record_offset, record_len) = first_block_record(volume);
let block_size = record_len - BLOCK_RECORD_FRAMING_LEN;
let mut record = BlockRecord::parse(
&volume[record_offset..record_offset + record_len],
block_size,
)
.unwrap();
mutate(&mut record);
volume[record_offset..record_offset + record_len].copy_from_slice(&record.to_bytes());
}
fn first_block_record(volume: &[u8]) -> (usize, usize) {
let volume_header = VolumeHeader::parse(&volume[..VOLUME_HEADER_LEN]).unwrap();
let crypto_start = volume_header.crypto_header_offset as usize;
let crypto_end = crypto_start + volume_header.crypto_header_length as usize;
let crypto_header = CryptoHeader::parse(
&volume[crypto_start..crypto_end],
volume_header.crypto_header_length,
)
.unwrap();
let record_offset = crypto_end;
let record_len = crypto_header.fixed.block_size as usize + BLOCK_RECORD_FRAMING_LEN;
assert!(volume.len() >= record_offset + record_len);
(record_offset, record_len)
}
fn rewrite_sidecar_header(
sidecar: &mut [u8],
master_key: &MasterKey,
mutate: impl FnOnce(&mut BootstrapSidecarHeader),
) {
let mut header =
BootstrapSidecarHeader::parse(&sidecar[..BOOTSTRAP_SIDECAR_HEADER_LEN]).unwrap();
mutate(&mut header);
header.sidecar_hmac = [0u8; 32];
let mut header_bytes = header.to_bytes();
let subkeys =
Subkeys::derive(master_key, &header.archive_uuid, &header.session_id).unwrap();
header.sidecar_hmac = compute_hmac(
HmacDomain::BootstrapSidecar,
&subkeys.mac_key,
&header.archive_uuid,
&header.session_id,
&header_bytes[..SIDECAR_HMAC_COVERED_LEN],
);
header_bytes = header.to_bytes();
sidecar[..BOOTSTRAP_SIDECAR_HEADER_LEN].copy_from_slice(&header_bytes);
}
fn mutate_sidecar_manifest(
sidecar: &mut [u8],
master_key: &MasterKey,
mutate: impl FnOnce(&mut ManifestFooter),
) {
let header =
BootstrapSidecarHeader::parse(&sidecar[..BOOTSTRAP_SIDECAR_HEADER_LEN]).unwrap();
let offset = header.manifest_footer_offset as usize;
let mut footer =
ManifestFooter::parse(&sidecar[offset..offset + MANIFEST_FOOTER_LEN]).unwrap();
mutate(&mut footer);
footer.manifest_hmac = [0u8; 32];
let mut footer_bytes = footer.to_bytes();
let subkeys =
Subkeys::derive(master_key, &footer.archive_uuid, &footer.session_id).unwrap();
footer.manifest_hmac = compute_hmac(
HmacDomain::ManifestFooter,
&subkeys.mac_key,
&footer.archive_uuid,
&footer.session_id,
&footer_bytes[..MANIFEST_HMAC_COVERED_LEN],
);
footer_bytes = footer.to_bytes();
sidecar[offset..offset + MANIFEST_FOOTER_LEN].copy_from_slice(&footer_bytes);
}
fn mutate_sidecar_index_record(
sidecar: &mut [u8],
record_index: usize,
mutate: impl FnOnce(&mut BlockRecord),
) {
let header =
BootstrapSidecarHeader::parse(&sidecar[..BOOTSTRAP_SIDECAR_HEADER_LEN]).unwrap();
let record_len = sidecar_record_len(sidecar);
let offset = header.index_root_records_offset as usize + record_index * record_len;
let block_size = record_len - BLOCK_RECORD_FRAMING_LEN;
let mut record =
BlockRecord::parse(&sidecar[offset..offset + record_len], block_size).unwrap();
mutate(&mut record);
sidecar[offset..offset + record_len].copy_from_slice(&record.to_bytes());
}
fn mutate_sidecar_dictionary_record(
sidecar: &mut [u8],
record_index: usize,
mutate: impl FnOnce(&mut BlockRecord),
) {
let header =
BootstrapSidecarHeader::parse(&sidecar[..BOOTSTRAP_SIDECAR_HEADER_LEN]).unwrap();
let record_len = sidecar_record_len(sidecar);
let offset = header.dictionary_records_offset as usize + record_index * record_len;
let block_size = record_len - BLOCK_RECORD_FRAMING_LEN;
let mut record =
BlockRecord::parse(&sidecar[offset..offset + record_len], block_size).unwrap();
mutate(&mut record);
sidecar[offset..offset + record_len].copy_from_slice(&record.to_bytes());
}
fn swap_sidecar_index_records(sidecar: &mut [u8], left: usize, right: usize) {
let header =
BootstrapSidecarHeader::parse(&sidecar[..BOOTSTRAP_SIDECAR_HEADER_LEN]).unwrap();
let record_len = sidecar_record_len(sidecar);
let left_offset = header.index_root_records_offset as usize + left * record_len;
let right_offset = header.index_root_records_offset as usize + right * record_len;
for idx in 0..record_len {
sidecar.swap(left_offset + idx, right_offset + idx);
}
}
fn sidecar_record_len(sidecar: &[u8]) -> usize {
let header =
BootstrapSidecarHeader::parse(&sidecar[..BOOTSTRAP_SIDECAR_HEADER_LEN]).unwrap();
let footer_offset = header.manifest_footer_offset as usize;
let footer =
ManifestFooter::parse(&sidecar[footer_offset..footer_offset + MANIFEST_FOOTER_LEN])
.unwrap();
let index_record_count = footer.index_root_data_block_count as usize
+ footer.index_root_parity_block_count as usize;
header.index_root_records_length as usize / index_record_count
}
}