use crate::storage::{Storage, StorageBackend};
use crate::{Error, Lockbox, LockboxOptions, ReadOnly, SecretVec};
use std::collections::BTreeMap;
use std::fmt;
use std::sync::{Arc, Mutex};
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SourceError {
MissingRange {
offset: u64,
length: usize,
},
VersionChanged,
Cancelled,
Unavailable(String),
}
impl fmt::Display for SourceError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{self:?}")
}
}
impl std::error::Error for SourceError {}
pub trait ReadAtSource: Send + Sync + 'static {
fn len(&self) -> u64;
fn is_empty(&self) -> bool {
self.len() == 0
}
fn read_at(&self, offset: u64, output: &mut [u8]) -> Result<usize, SourceError>;
fn validate(&self) -> Result<(), SourceError> {
Ok(())
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ExternalReadError {
Source(SourceError),
Archive(Error),
RetryLimitExceeded,
}
impl fmt::Display for ExternalReadError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{self:?}")
}
}
impl std::error::Error for ExternalReadError {}
#[derive(Debug, Clone, Copy)]
pub struct ExternalReaderOptions {
pub max_read_bytes: usize,
pub max_missing_attempts: usize,
pub max_source_bytes_per_operation: u64,
pub lockbox: LockboxOptions,
}
impl Default for ExternalReaderOptions {
fn default() -> Self {
Self {
max_read_bytes: 64 * 1024 * 1024,
max_missing_attempts: 4096,
max_source_bytes_per_operation: 256 * 1024 * 1024,
lockbox: LockboxOptions::default(),
}
}
}
pub struct ExternalReader {
storage: ExternalStorage,
archive: Option<Lockbox<ReadOnly>>,
key: Option<SecretVec>,
options: ExternalReaderOptions,
missing_attempts: usize,
terminal: Option<ExternalReadError>,
}
impl ExternalReader {
pub fn new(
source: Arc<dyn ReadAtSource>,
options: ExternalReaderOptions,
) -> Result<Self, ExternalReadError> {
if source.is_empty()
|| options.max_read_bytes == 0
|| options.max_missing_attempts == 0
|| options.max_source_bytes_per_operation == 0
{
return Err(ExternalReadError::Archive(Error::InvalidInput(
"external source length and limits must be nonzero".into(),
)));
}
Ok(Self {
storage: ExternalStorage {
length: source.len(),
source,
failure: Arc::new(Mutex::new(None)),
max_read_bytes: options.max_read_bytes,
max_operation_bytes: options.max_source_bytes_per_operation,
operation_bytes: Arc::new(Mutex::new(0)),
},
archive: None,
key: None,
options,
missing_attempts: 0,
terminal: None,
})
}
pub fn with_content_key(
source: Arc<dyn ReadAtSource>,
key: SecretVec,
options: ExternalReaderOptions,
) -> Result<Self, ExternalReadError> {
let mut reader = Self::new(source, options)?;
reader.key = Some(key);
Ok(reader)
}
pub fn cancel(&mut self) {
self.archive = None;
self.key = None;
self.terminal = Some(ExternalReadError::Source(SourceError::Cancelled));
}
pub fn try_read<T>(
&mut self,
operation: impl FnOnce(&Lockbox<ReadOnly>) -> crate::Result<T>,
) -> Result<T, ExternalReadError> {
if let Some(error) = &self.terminal {
return Err(error.clone());
}
self.storage
.clear_failure()
.map_err(ExternalReadError::Source)?;
let validation = self.storage.source.validate().and_then(|()| {
if self.storage.source.len() == self.storage.length {
Ok(())
} else {
Err(SourceError::VersionChanged)
}
});
if let Err(error) = validation {
return self.fail_source(error);
}
let mut candidate = self.archive.take();
let result = (|| {
if candidate.is_none() {
let key = match &self.key {
Some(key) => key.try_clone()?,
None => {
let header = self.storage.read_at(0, crate::constants::HEADER_LEN)?;
let header = crate::file_format::current_header::read_header(&header)?;
if !header.format_mode.plaintext() {
return Err(Error::InvalidInput(
"this external archive requires a content key".into(),
));
}
SecretVec::try_from_slice(&[0; 32])?
}
};
let archive = Lockbox::open_storage_with_secret_key_mode(
StorageBackend::External(self.storage.clone()),
key,
self.options.lockbox,
false,
)?;
if self.storage.failure().map_err(source_io)?.is_some() {
return Err(Error::Io("incomplete external open".into()));
}
candidate = Some(archive.into_state());
}
operation(
candidate
.as_ref()
.ok_or_else(|| Error::InvalidOperation("external archive not open".into()))?,
)
})();
if let Err(error) = self.storage.source.validate() {
return self.fail_source(error);
}
if self.storage.source.len() != self.storage.length {
return self.fail_source(SourceError::VersionChanged);
}
if let Some(error) = self.storage.failure().map_err(ExternalReadError::Source)? {
return self.fail_source(error);
}
self.missing_attempts = 0;
*self
.storage
.operation_bytes
.lock()
.map_err(|_| ExternalReadError::Source(poisoned()))? = 0;
if result.is_ok() {
self.archive = candidate;
}
result.map_err(ExternalReadError::Archive)
}
fn fail_source<T>(&mut self, error: SourceError) -> Result<T, ExternalReadError> {
self.archive = None;
let error = match error {
SourceError::MissingRange { offset, length }
if length == 0
|| length > self.storage.max_read_bytes
|| offset
.checked_add(length as u64)
.is_none_or(|end| end > self.storage.length) =>
{
SourceError::Unavailable(
"external source requested an invalid missing range".into(),
)
}
other => other,
};
let failure = if matches!(error, SourceError::MissingRange { .. }) {
self.missing_attempts += 1;
if self.missing_attempts >= self.options.max_missing_attempts {
ExternalReadError::RetryLimitExceeded
} else {
ExternalReadError::Source(error)
}
} else {
ExternalReadError::Source(error)
};
if !matches!(
failure,
ExternalReadError::Source(SourceError::MissingRange { .. })
) {
self.key = None;
self.terminal = Some(failure.clone());
}
Err(failure)
}
}
fn source_io(error: SourceError) -> Error {
Error::Io(error.to_string())
}
fn poisoned() -> SourceError {
SourceError::Unavailable("external source state poisoned".into())
}
#[derive(Clone)]
pub(crate) struct ExternalStorage {
source: Arc<dyn ReadAtSource>,
length: u64,
failure: Arc<Mutex<Option<SourceError>>>,
max_read_bytes: usize,
max_operation_bytes: u64,
operation_bytes: Arc<Mutex<u64>>,
}
impl fmt::Debug for ExternalStorage {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ExternalStorage")
.field("length", &self.length)
.finish_non_exhaustive()
}
}
impl ExternalStorage {
pub(crate) fn ensure_current(&self) -> crate::Result<()> {
if let Some(error) = self.failure().map_err(source_io)? {
return Err(source_io(error));
}
self.source.validate().map_err(|error| self.latch(error))?;
if self.source.len() != self.length {
return Err(self.latch(SourceError::VersionChanged));
}
Ok(())
}
fn failure(&self) -> Result<Option<SourceError>, SourceError> {
Ok(self.failure.lock().map_err(|_| poisoned())?.clone())
}
fn clear_failure(&self) -> Result<(), SourceError> {
*self.failure.lock().map_err(|_| poisoned())? = None;
Ok(())
}
fn latch(&self, error: SourceError) -> Error {
if let Ok(mut failure) = self.failure.lock() {
if failure.is_none() {
*failure = Some(error.clone());
}
}
source_io(error)
}
fn bounds(&self, offset: u64, len: usize) -> crate::Result<()> {
if len > self.max_read_bytes {
return Err(self.latch(SourceError::Unavailable(
"external read exceeds configured byte limit".into(),
)));
}
if offset
.checked_add(len as u64)
.is_none_or(|end| end > self.length)
{
return Err(self.latch(SourceError::Unavailable(
"external read exceeds source length".into(),
)));
}
Ok(())
}
}
impl Storage for ExternalStorage {
fn len(&self) -> crate::Result<u64> {
Ok(self.length)
}
fn read_at(&self, offset: u64, len: usize) -> crate::Result<Vec<u8>> {
self.bounds(offset, len)?;
let mut bytes = vec![0; len];
self.read_at_into(offset, &mut bytes)?;
Ok(bytes)
}
fn read_at_into(&self, offset: u64, out: &mut [u8]) -> crate::Result<()> {
if let Some(error) = self.failure().map_err(source_io)? {
return Err(source_io(error));
}
self.bounds(offset, out.len())?;
if out.is_empty() {
return Ok(());
}
{
let mut spent = self
.operation_bytes
.lock()
.map_err(|_| self.latch(poisoned()))?;
let next = spent
.checked_add(out.len() as u64)
.filter(|value| *value <= self.max_operation_bytes);
let Some(next) = next else {
return Err(self.latch(SourceError::Unavailable(
"external operation exceeds cumulative read budget".into(),
)));
};
*spent = next;
}
let result = self.source.read_at(offset, out);
match result {
Ok(count) if count == out.len() => Ok(()),
Ok(_) => Err(self.latch(SourceError::Unavailable(
"external source returned a short or oversized read".into(),
))),
Err(SourceError::MissingRange { offset, length })
if length == 0
|| length > self.max_read_bytes
|| offset
.checked_add(length as u64)
.is_none_or(|end| end > self.length) =>
{
Err(self.latch(SourceError::Unavailable(
"external source requested an invalid missing range".into(),
)))
}
Err(error) => Err(self.latch(error)),
}
}
fn read_at_secure(
&self,
offset: u64,
len: usize,
) -> crate::Result<crate::secret_vec::SecureVec> {
self.bounds(offset, len)?;
let mut out = crate::secret_vec::SecureVec::new();
out.resize_zeroed(len)?;
out.with_mut_bytes(|bytes| self.read_at_into(offset, bytes))??;
Ok(out)
}
fn append(&mut self, _: &[u8]) -> crate::Result<u64> {
Err(Error::InvalidOperation(
"external archive is read-only".into(),
))
}
fn write_at(&mut self, _: u64, _: &[u8]) -> crate::Result<()> {
Err(Error::InvalidOperation(
"external archive is read-only".into(),
))
}
fn truncate(&mut self, _: u64) -> crate::Result<()> {
Err(Error::InvalidOperation(
"external archive is read-only".into(),
))
}
fn sync(&self) -> crate::Result<()> {
Ok(())
}
}
#[derive(Clone)]
pub struct SparseSource {
length: u64,
revision: Arc<str>,
block_bytes: usize,
capacity_bytes: usize,
state: Arc<Mutex<SparseState>>,
}
#[derive(Default)]
struct SparseState {
blocks: BTreeMap<u64, (Vec<u8>, u64)>,
bytes: usize,
clock: u64,
terminal: Option<SourceError>,
}
impl SparseState {
fn tick(&mut self) -> u64 {
if self.clock == u64::MAX {
for (_, age) in self.blocks.values_mut() {
*age = 0;
}
self.clock = 0;
}
self.clock += 1;
self.clock
}
}
impl SparseSource {
pub fn new(
length: u64,
revision: String,
block_bytes: usize,
capacity_bytes: usize,
) -> Result<Self, SourceError> {
if length == 0 || revision.is_empty() || block_bytes < 512 || capacity_bytes < block_bytes {
return Err(SourceError::Unavailable(
"invalid sparse length, revision, block size or budget".into(),
));
}
Ok(Self {
length,
revision: revision.into(),
block_bytes,
capacity_bytes,
state: Arc::new(Mutex::new(SparseState::default())),
})
}
pub fn supply(&self, revision: &str, offset: u64, bytes: Vec<u8>) -> Result<(), SourceError> {
let mut state = self.state.lock().map_err(|_| poisoned())?;
if let Some(error) = &state.terminal {
return Err(error.clone());
}
if revision != self.revision.as_ref() {
state.blocks.clear();
state.bytes = 0;
state.terminal = Some(SourceError::VersionChanged);
return Err(SourceError::VersionChanged);
}
if offset >= self.length
|| offset % self.block_bytes as u64 != 0
|| bytes.len() != (self.length - offset).min(self.block_bytes as u64) as usize
{
return Err(SourceError::Unavailable(
"expected one exact aligned sparse block".into(),
));
}
if let Some((previous, _)) = state.blocks.get(&offset) {
if previous != &bytes {
state.blocks.clear();
state.bytes = 0;
state.terminal = Some(SourceError::VersionChanged);
return Err(SourceError::VersionChanged);
}
let age = state.tick();
if let Some((_, previous_age)) = state.blocks.get_mut(&offset) {
*previous_age = age;
}
return Ok(());
}
while state.bytes > self.capacity_bytes - bytes.len() {
let oldest = state
.blocks
.iter()
.min_by_key(|(_, (_, age))| *age)
.map(|(&offset, _)| offset)
.ok_or_else(|| {
SourceError::Unavailable("invalid sparse cache accounting".into())
})?;
if let Some((removed, _)) = state.blocks.remove(&oldest) {
state.bytes -= removed.len();
}
}
let age = state.tick();
state.bytes += bytes.len();
state.blocks.insert(offset, (bytes, age));
Ok(())
}
pub fn cached_bytes(&self) -> Result<usize, SourceError> {
Ok(self.state.lock().map_err(|_| poisoned())?.bytes)
}
pub fn cancel(&self) -> Result<(), SourceError> {
let mut state = self.state.lock().map_err(|_| poisoned())?;
state.blocks.clear();
state.bytes = 0;
state.terminal = Some(SourceError::Cancelled);
Ok(())
}
}
impl ReadAtSource for SparseSource {
fn len(&self) -> u64 {
self.length
}
fn validate(&self) -> Result<(), SourceError> {
match &self.state.lock().map_err(|_| poisoned())?.terminal {
Some(error) => Err(error.clone()),
None => Ok(()),
}
}
fn read_at(&self, offset: u64, output: &mut [u8]) -> Result<usize, SourceError> {
let end = offset
.checked_add(output.len() as u64)
.filter(|end| *end <= self.length)
.ok_or_else(|| SourceError::Unavailable("sparse read outside source".into()))?;
let mut state = self.state.lock().map_err(|_| poisoned())?;
if let Some(error) = &state.terminal {
return Err(error.clone());
}
let mut position = offset;
while position < end {
let start = position / self.block_bytes as u64 * self.block_bytes as u64;
let age = state.tick();
let (bytes, last_used) =
state
.blocks
.get_mut(&start)
.ok_or(SourceError::MissingRange {
offset: start,
length: (self.length - start).min(self.block_bytes as u64) as usize,
})?;
let finish = end.min(start + bytes.len() as u64);
output[(position - offset) as usize..(finish - offset) as usize]
.copy_from_slice(&bytes[(position - start) as usize..(finish - start) as usize]);
*last_used = age;
position = finish;
}
Ok(output.len())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[cfg(feature = "native-block-layout")]
#[test]
fn small_native_whole_reads_capture_once_and_preserve_source_failures() {
use crate::{
Compression, Encryption, LockboxCreateOptions, LockboxProtection, OwnerSigningKeyPair,
Signing, SizePadding,
};
use std::io::Read;
struct Probe {
bytes: Vec<u8>,
reads: Mutex<usize>,
cancel_on_read: bool,
cancelled: Mutex<bool>,
}
impl ReadAtSource for Probe {
fn len(&self) -> u64 {
self.bytes.len() as u64
}
fn validate(&self) -> Result<(), SourceError> {
if *self.cancelled.lock().unwrap() {
Err(SourceError::Cancelled)
} else {
Ok(())
}
}
fn read_at(&self, offset: u64, out: &mut [u8]) -> Result<usize, SourceError> {
*self.reads.lock().unwrap() += 1;
out.copy_from_slice(&self.bytes[offset as usize..offset as usize + out.len()]);
if self.cancel_on_read {
*self.cancelled.lock().unwrap() = true;
}
Ok(out.len())
}
}
let signer = OwnerSigningKeyPair::generate().unwrap();
let input = (0..4096).map(|n| (n % 251) as u8).collect::<Vec<_>>();
let path = crate::LockboxPath::new("/small").unwrap();
for encrypted in [false, true] {
for signed in [false, true] {
for compression in [Compression::None, Compression::default()] {
for size_padding in [SizePadding::Default, SizePadding::None] {
let mut archive =
crate::Lockbox::create_in_memory_with_options(LockboxCreateOptions {
compression,
size_padding,
..LockboxCreateOptions::new(
if encrypted {
Encryption::Encrypted(LockboxProtection::ContentKey(
crate::SecretVec::try_from_slice(&[67; 32]).unwrap(),
))
} else {
Encryption::None
},
if signed {
Signing::Owner(&signer)
} else {
Signing::None
},
)
})
.unwrap();
archive.add_file(&path, &input, false).unwrap();
archive.commit().unwrap();
let bytes = archive.to_bytes();
for case in 0..8 {
let through_handle = case >= 4;
let fault = case % 4;
let mut source_bytes = bytes.clone();
if fault >= 2 {
let offset = source_bytes.windows(crate::page::PAGE_HEADER_LEN).position(crate::file_format::indexed_frame::block_page::is_native_header).unwrap();
if fault == 2 {
source_bytes[offset + crate::page::PAGE_HEADER_LEN + 4] ^= 1;
} else {
let len = crate::page::physical_page_size_from_page_slice(
&source_bytes[offset..],
)
.unwrap();
source_bytes.truncate(offset + len - 1);
}
}
let source = Arc::new(Probe {
bytes: source_bytes,
reads: Mutex::new(0),
cancel_on_read: fault == 1,
cancelled: Mutex::new(false),
});
let session = ExternalReader::new(
source.clone(),
ExternalReaderOptions::default(),
)
.unwrap();
crate::lockbox::block_frame_tests::replace_storage(
&mut archive,
crate::storage::StorageBackend::External(session.storage.clone()),
);
let read = || -> crate::Result<Vec<u8>> {
if !through_handle {
return archive.get_file(&path);
}
let mut handle = archive.open_file(&path)?;
let mut bytes = Vec::new();
handle
.read_to_end(&mut bytes)
.map_err(|e| crate::Error::Io(e.to_string()))?;
Ok(bytes)
};
if fault == 0 {
assert_eq!(read().unwrap(), input);
assert_eq!(
*source.reads.lock().unwrap(),
1,
"one bounded source read per cold small frame"
);
assert_eq!(read().unwrap(), input);
assert_eq!(
*source.reads.lock().unwrap(),
1,
"warm read reuses verified cache"
);
} else if fault == 1 {
assert!(read().is_err());
*source.cancelled.lock().unwrap() = false;
assert!(
read().is_err(),
"terminal source failure must stay latched"
);
assert_eq!(*source.reads.lock().unwrap(), 1);
} else {
assert!(read().is_err());
}
}
}
}
}
}
}
#[test]
fn native_file_handle_retains_index_and_bounds_raw_reads_without_bypassing_failure_latch() {
use crate::{
Compression, Encryption, LockboxCreateOptions, LockboxProtection, OwnerSigningKeyPair,
Signing, SizePadding,
};
use std::io::{Read, Seek, SeekFrom};
struct Probe {
bytes: Vec<u8>,
cancelled: Mutex<bool>,
reads: Mutex<Vec<(u64, usize)>>,
}
impl ReadAtSource for Probe {
fn len(&self) -> u64 {
self.bytes.len() as u64
}
fn validate(&self) -> Result<(), SourceError> {
if *self.cancelled.lock().unwrap() {
Err(SourceError::Cancelled)
} else {
Ok(())
}
}
fn read_at(&self, offset: u64, out: &mut [u8]) -> Result<usize, SourceError> {
self.reads.lock().unwrap().push((offset, out.len()));
out.copy_from_slice(&self.bytes[offset as usize..offset as usize + out.len()]);
Ok(out.len())
}
}
let signer = OwnerSigningKeyPair::generate().unwrap();
let input = (0..2 * 1024 * 1024 + 13)
.map(|n| (n % 251) as u8)
.collect::<Vec<_>>();
for encrypted in [false, true] {
for signed in [false, true] {
for compression in [Compression::None, Compression::default()] {
for size_padding in [SizePadding::Default, SizePadding::None] {
let mut archive =
Lockbox::create_in_memory_with_options(LockboxCreateOptions {
compression,
size_padding,
..LockboxCreateOptions::new(
if encrypted {
Encryption::Encrypted(LockboxProtection::ContentKey(
SecretVec::try_from_slice(&[67; 32]).unwrap(),
))
} else {
Encryption::None
},
if signed {
Signing::Owner(&signer)
} else {
Signing::None
},
)
})
.unwrap();
archive.commit().unwrap();
let paths =
crate::lockbox::block_frame_tests::install(&mut archive, &input);
let expected = &input[input.len() / 2..];
let source = Arc::new(Probe {
bytes: archive.to_bytes(),
cancelled: Mutex::new(false),
reads: Mutex::new(Vec::new()),
});
let session =
ExternalReader::new(source.clone(), ExternalReaderOptions::default())
.unwrap();
crate::lockbox::block_frame_tests::replace_storage(
&mut archive,
StorageBackend::External(session.storage.clone()),
);
archive.set_workload_profile(crate::WorkloadProfile::ExtractMany);
let mut reader = archive.open_file(&paths[1]).unwrap();
reader.seek(SeekFrom::Start(7)).unwrap();
let mut small = [0; 13];
reader.read_exact(&mut small).unwrap();
assert_eq!(small, expected[7..20]);
let initial = source.reads.lock().unwrap().clone();
assert_eq!(initial.len(), 4, "header, metadata, index, data");
if compression == Compression::None {
assert_eq!(initial[3].1, 16384 + if encrypted { 16 } else { 0 });
}
reader.seek(SeekFrom::Start(7)).unwrap();
reader.read_exact(&mut small).unwrap();
assert_eq!(small, expected[7..20]);
assert_eq!(*source.reads.lock().unwrap(), initial);
let mut second = archive.open_file(&paths[1]).unwrap();
second.seek(SeekFrom::Start(7)).unwrap();
second.read_exact(&mut small).unwrap();
assert_eq!(small, expected[7..20]);
assert_eq!(*source.reads.lock().unwrap(), initial);
drop(second);
reader.seek(SeekFrom::Start(32770)).unwrap();
reader.read_exact(&mut small).unwrap();
assert_eq!(small, expected[32770..32783]);
assert_eq!(
source.reads.lock().unwrap().len(),
initial.len() + usize::from(compression == Compression::None)
);
reader.rewind().unwrap();
let mut actual = Vec::new();
let mut buffer = [0; 65536];
loop {
let count = reader.read(&mut buffer).unwrap();
if count == 0 {
break;
}
actual.extend_from_slice(&buffer[..count]);
}
assert_eq!(actual, expected);
assert!(
source.reads.lock().unwrap()[4..]
.iter()
.all(|(offset, _)| *offset >= initial[3].0),
"sequential reads must not reload metadata or the index"
);
if compression != Compression::None {
assert_eq!(
source.reads.lock().unwrap().len(),
4,
"decode once per retained chunk"
);
}
reader.seek(SeekFrom::End(-13)).unwrap();
reader.read_exact(&mut small).unwrap();
if compression == Compression::None {
let before = source.reads.lock().unwrap().len();
let mut neighbor = archive.open_file(&paths[0]).unwrap();
let mut bytes = [0; 13];
neighbor.read_exact(&mut bytes).unwrap();
assert_eq!(bytes, input[..13]);
let reads = source.reads.lock().unwrap();
assert_eq!(
reads.len(),
before + 1,
"new raw handles reuse metadata/index but verify requested data"
);
assert_eq!(reads[before].1, 16384 + if encrypted { 16 } else { 0 });
}
assert_eq!(
archive.get_file(&paths[0]).unwrap(),
input[..input.len() / 2]
);
assert_eq!(
archive.decoded_compression_frame_cache_entries_for_tests(),
1
);
let before_cached = source.reads.lock().unwrap().len();
assert_eq!(
archive.read_file_range(&paths[1], 7, 13).unwrap(),
expected[7..20]
);
assert_eq!(source.reads.lock().unwrap().len(), before_cached,
"verified decoded cache hit reuses metadata and index without source reads");
if compression != Compression::None {
let before = source.reads.lock().unwrap().len();
let mut neighbor = archive.open_file(&paths[0]).unwrap();
let mut bytes = Vec::new();
neighbor.read_to_end(&mut bytes).unwrap();
assert_eq!(bytes, input[..input.len() / 2]);
assert_eq!(
source.reads.lock().unwrap().len(),
before,
"new compressed handles reuse the verified shared frame"
);
}
let count = source.reads.lock().unwrap().len();
*source.cancelled.lock().unwrap() = true;
reader.seek(SeekFrom::End(-13)).unwrap();
assert!(reader.read_exact(&mut small).is_err());
assert!(archive.get_file(&paths[0]).is_err());
*source.cancelled.lock().unwrap() = false;
assert!(archive.read_file_range(&paths[1], 7, 13).is_err());
assert!(
reader.read_exact(&mut small).is_err(),
"cache cannot clear terminal failure"
);
assert_eq!(source.reads.lock().unwrap().len(), count);
drop(reader);
let session =
ExternalReader::new(source.clone(), ExternalReaderOptions::default())
.unwrap();
crate::lockbox::block_frame_tests::replace_storage(
&mut archive,
StorageBackend::External(session.storage.clone()),
);
use crate::{
ContentStreamOrder::{Logical, Physical},
WorkloadProfile::{ExtractMany, Interactive},
};
for (profile, order, expected_reads) in [
(Interactive, Logical, 1),
(Interactive, Physical, 1),
(ExtractMany, Logical, 1),
(ExtractMany, Physical, 3),
] {
archive.set_workload_profile(profile);
if matches!(profile, Interactive) {
let mut visited = 0;
assert!(archive
.stream_content(
crate::ContentStreamOptions { order },
|_, _| {
visited += 1;
Err(crate::Error::Io("visitor stopped".into()))
}
)
.is_err());
assert_eq!(visited, 1);
}
source.reads.lock().unwrap().clear();
let mut actual = Vec::new();
archive
.stream_content(
crate::ContentStreamOptions { order },
|_, reader| {
reader.read_to_end(&mut actual).unwrap();
Ok(())
},
)
.unwrap();
assert_eq!(actual, input);
assert_eq!(source.reads.lock().unwrap().len(), expected_reads,
"packed stream reuses retained index; decoded-cache replacement reloads only metadata/index");
}
let mut visited = 0;
let result = archive.stream_content(Default::default(), |_, _| {
visited += 1;
*source.cancelled.lock().unwrap() = true;
Ok(())
});
assert!(result.is_err());
assert_eq!(
visited, 1,
"cancelled source cannot deliver its retained neighbor"
);
*source.cancelled.lock().unwrap() = false;
assert!(archive
.stream_content(Default::default(), |_, _| Ok(()))
.is_err());
}
}
}
}
}
#[test]
fn native_block_reader_preserves_external_failure_latching_in_every_mode() {
use crate::file_format::indexed_frame::{encode_block_frame, BlockFrameReader};
use crate::{Compression, EncryptionMode, LockboxFormatOptions, SigningMode, SizePadding};
struct Probe {
bytes: Vec<u8>,
state: Mutex<u8>,
reads: Mutex<Vec<(u64, usize)>>,
}
impl ReadAtSource for Probe {
fn len(&self) -> u64 {
self.bytes.len() as u64 + u64::from(*self.state.lock().unwrap() == 5)
}
fn validate(&self) -> Result<(), SourceError> {
match *self.state.lock().unwrap() {
1 => Err(SourceError::Cancelled),
2 => Err(SourceError::VersionChanged),
_ => Ok(()),
}
}
fn read_at(&self, offset: u64, out: &mut [u8]) -> Result<usize, SourceError> {
self.reads.lock().unwrap().push((offset, out.len()));
let state = *self.state.lock().unwrap();
if state == 4 {
return Err(SourceError::MissingRange {
offset,
length: out.len(),
});
}
out.copy_from_slice(&self.bytes[offset as usize..offset as usize + out.len()]);
Ok(out.len() - usize::from(state == 3 && !out.is_empty()))
}
}
let key = [29; 32];
for encrypted in [false, true] {
for signed in [false, true] {
for compressed in [false, true] {
let mode = crate::creation_options::FormatMode::new(LockboxFormatOptions {
encryption: if encrypted {
EncryptionMode::ChaCha20Poly1305
} else {
EncryptionMode::None
},
signing: if signed {
SigningMode::Owner
} else {
SigningMode::None
},
compression: if compressed {
Compression::default()
} else {
Compression::None
},
size_padding: SizePadding::Default,
});
let (descriptor, packet) = encode_block_frame(
crate::LockboxId::from_bytes([71; 16]),
31,
mode,
&[43; 32768],
&key,
)
.unwrap();
for failure in 1..=5 {
let source = Arc::new(Probe {
bytes: packet.clone(),
state: Mutex::new(0),
reads: Mutex::new(Vec::new()),
});
let session =
ExternalReader::new(source.clone(), ExternalReaderOptions::default())
.unwrap();
let backend = StorageBackend::External(session.storage.clone());
let reader =
BlockFrameReader::open(&descriptor, &backend, 0, packet.len(), &key)
.unwrap();
assert_eq!(
source.reads.lock().unwrap().len(),
1,
"open must read only the committed index"
);
assert_eq!(reader.read(7..19).unwrap(), [43; 12]);
assert_eq!(source.reads.lock().unwrap().len(), 2);
if !compressed {
assert_eq!(
source.reads.lock().unwrap()[1].1,
16384 + if encrypted { 16 } else { 0 }
);
}
*source.state.lock().unwrap() = failure;
if !encrypted && !compressed {
let mut direct = vec![88; 32768];
assert!(reader.read_aligned_raw_into(0..32768, &mut direct).is_err());
assert_eq!(
direct,
vec![0; 32768],
"direct I/O failures wipe the entire destination"
);
}
assert!(reader.read(7..19).is_err());
let latched = session.storage.failure().unwrap().unwrap();
match failure {
1 => assert_eq!(latched, SourceError::Cancelled),
2 | 5 => assert_eq!(latched, SourceError::VersionChanged),
3 => assert!(matches!(latched, SourceError::Unavailable(_))),
4 => assert!(matches!(latched, SourceError::MissingRange { .. })),
_ => unreachable!(),
}
*source.state.lock().unwrap() = 0;
let reads_before_retry = source.reads.lock().unwrap().len();
assert!(reader.read(0..0).is_err());
assert!(reader.read(7..19).is_err());
assert_eq!(source.reads.lock().unwrap().len(), reads_before_retry);
if failure == 4 {
session.storage.clear_failure().unwrap();
assert_eq!(reader.read(7..19).unwrap(), [43; 12]);
}
}
}
}
}
}
#[test]
fn backend_refuses_every_mutation_and_checks_limits_before_secure_allocation() {
let source = Arc::new(SparseSource::new(4096, "v1".into(), 512, 512).unwrap());
let mut storage = ExternalStorage {
source,
length: 4096,
max_read_bytes: 1024,
max_operation_bytes: 8192,
operation_bytes: Arc::new(Mutex::new(0)),
failure: Arc::new(Mutex::new(None)),
};
assert!(storage.append(b"x").is_err());
assert!(storage.write_at(0, b"x").is_err());
assert!(storage.truncate(0).is_err());
assert!(StorageBackend::External(storage.clone())
.read_at_secure(0, usize::MAX)
.is_err());
assert!(matches!(
storage.failure().unwrap(),
Some(SourceError::Unavailable(_))
));
assert!(StorageBackend::External(storage).is_read_only());
}
}