use super::*;
use std::path::PathBuf;
#[path = "set_verify.rs"]
mod set_verify;
pub use set_verify::{SetVerification, SetVolumeVerification};
#[derive(Debug, Clone, serde::Serialize)]
pub struct VolumeSource {
pub stream: String,
pub volume: String,
pub path: PathBuf,
}
#[derive(Debug, Clone, serde::Serialize)]
pub struct SetDigest {
pub image: String,
pub bytes: u64,
pub sha256: String,
pub external_match: Option<bool>,
pub sources: Vec<VolumeSource>,
}
pub struct VolumeSet {
volumes: Vec<Container>,
paths: Vec<PathBuf>,
owners: BTreeMap<String, usize>,
validated: BTreeMap<(usize, String), (Arc<Map>, u64)>,
cached_owner: Option<usize>,
}
impl VolumeSet {
pub fn open(paths: &[PathBuf]) -> Result<Self> {
Self::open_with_limits(paths, Limits::default())
}
pub fn open_with_limits(paths: &[PathBuf], limits: Limits) -> Result<Self> {
if paths.is_empty() || paths.len() > 128 {
return Err(malformed("volume set must contain 1 through 128 paths"));
}
let mut volumes = Vec::new();
let mut ids = BTreeSet::new();
let mut remaining = limits.clone();
for path in paths {
let mut volume = Container::open_with_limits(path, remaining.clone())?;
remaining.metadata_bytes -= volume.usage.metadata_bytes;
remaining.triples -= volume.usage.triples;
remaining.directory_bytes -= volume.usage.directory_bytes;
remaining.archive_entries -= volume.usage.entries;
volume.limits = limits.clone();
if volume.version != (1, 0) {
return Err(Error::Unsupported(
"volume sets currently support physical AFF4 1.0".into(),
));
}
if !ids.insert(volume.volume.clone()) {
return Err(malformed("duplicate volume identifier"));
}
volumes.push(volume);
}
Self::from_containers(volumes, paths.to_vec())
}
pub(super) fn from_containers(
mut volumes: Vec<Container>,
paths: Vec<PathBuf>,
) -> Result<Self> {
let ids: BTreeSet<_> = volumes.iter().map(|volume| volume.volume.clone()).collect();
if ids.len() != volumes.len() {
return Err(malformed("duplicate volume identifier"));
}
let mut owners = BTreeMap::new();
for (index, volume) in volumes.iter().enumerate() {
for id in volume
.graph
.keys()
.filter(|id| volume.has_type(id, "ImageStream"))
{
if let Some(stored) = volume.value(id, "stored")?
&& !ids.contains(&stored)
{
return Err(malformed(format!("missing source volume {stored}")));
}
let Ok(size) = volume.number(id, "size") else {
continue;
};
let path = volume.path(id)?;
let owns = if size == 0 {
volume.value(id, "stored")?.as_deref() == Some(&volume.volume)
} else {
volume
.archive
.index_for_name(&format!("{path}/00000000"))
.is_some()
};
if owns {
if volume
.value(id, "stored")?
.is_some_and(|v| v != volume.volume)
{
return Err(malformed("local stream conflicts with declared owner"));
}
if owners.insert(id.clone(), index).is_some() {
return Err(malformed(format!("ambiguous stream owner: {id}")));
}
}
}
}
for (id, &owner) in &owners {
for volume in &volumes {
if !volume.graph.contains_key(id) {
continue;
}
for key in [
"size",
"chunkSize",
"chunksInSegment",
"compressionMethod",
"stored",
] {
if let Some(value) = volume.value(id, key)? {
let expected = if key == "stored" {
Some(volumes[owner].volume.clone())
} else {
volumes[owner].value(id, key)?
};
if expected.as_ref() != Some(&value) {
return Err(malformed(format!("conflicting {key} for {id}")));
}
}
}
}
}
let decoded_cache = Arc::new(Mutex::new(DecodedChunkCache::new()));
for volume in &mut volumes {
volume.decoded_cache = Arc::clone(&decoded_cache);
}
Ok(Self {
volumes,
paths,
owners,
validated: BTreeMap::new(),
cached_owner: None,
})
}
pub fn images(&self) -> Vec<String> {
self.volumes[0]
.graph
.keys()
.filter(|id| self.volumes[0].has_type(id, "DiskImage"))
.cloned()
.collect()
}
pub fn sources(&self) -> Vec<VolumeSource> {
self.owners
.iter()
.map(|(id, &n)| VolumeSource {
stream: id.clone(),
volume: self.volumes[n].volume.clone(),
path: self.paths[n].clone(),
})
.collect()
}
pub fn reader_statistics(&self) -> ReaderStatistics {
self.volumes[0].reader_statistics()
}
pub fn reader_cache_info(&self) -> ReaderCacheInfo {
self.volumes[0].reader_cache_info()
}
fn image_map(&mut self, id: &str) -> Result<(Arc<Map>, u64)> {
self.image_map_in(0, id)
}
fn image_map_in(&mut self, primary: usize, id: &str) -> Result<(Arc<Map>, u64)> {
if let Some(result) = self.validated.get(&(primary, id.to_owned())) {
return Ok(result.clone());
}
if !self.volumes[primary].has_type(id, "DiskImage") {
return Err(Error::Unsupported("select a primary DiskImage".into()));
}
let target = self.volumes[primary]
.value(id, "dataStream")?
.ok_or_else(|| malformed("image has no dataStream"))?;
self.volumes[primary].load_map(&target)?;
let map = self.volumes[primary].maps[&target].clone();
self.volumes[primary].maps.clear();
let size = self.volumes[primary].number(&target, "size")?;
if self.volumes[primary].value(id, "size")?.is_some()
&& self.volumes[primary].number(id, "size")? != size
{
return Err(malformed("image/map size mismatch"));
}
let contiguous = self.volumes[primary].has_type(id, "ContiguousImage");
let mut end = 0;
for range in &map.ranges {
if contiguous && range.start != end {
return Err(malformed("gap in contiguous image"));
}
end = range.end;
if is_symbolic(&range.target) {
continue;
}
let owner = *self
.owners
.get(&range.target)
.ok_or_else(|| malformed(format!("missing owner for {}", range.target)))?;
let size = self.volumes[owner].number(&range.target, "size")?;
if range
.offset
.checked_add(range.end - range.start)
.is_none_or(|end| end > size)
{
return Err(malformed("map exceeds owned stream"));
}
}
if contiguous && end != size {
return Err(malformed("truncated contiguous map"));
}
if !is_symbolic(&map.gap) && !self.owners.contains_key(&map.gap) {
return Err(Error::Unsupported("external/nested gap stream".into()));
}
self.validated.clear();
self.validated
.insert((primary, id.into()), (map.clone(), size));
Ok((map, size))
}
pub(super) fn equivalent_disk_maps(
&mut self,
left: usize,
right: usize,
id: &str,
) -> Result<bool> {
let (left, size) = self.image_map_in(left, id)?;
let (right, other_size) = self.image_map_in(right, id)?;
if size != other_size {
return Ok(false);
}
let mut position = 0;
while position < size {
let (a, a_offset, a_end) = map_interval(&left, position, size);
let (b, b_offset, b_end) = map_interval(&right, position, size);
if a != b || (!is_symbolic(a) && a_offset != b_offset) {
return Ok(false);
}
position = a_end.min(b_end);
}
Ok(true)
}
pub fn size(&mut self, id: &str) -> Result<u64> {
self.image_map(id).map(|(_, size)| size)
}
pub fn read_at(&mut self, id: &str, buffer: &mut [u8], offset: u64) -> Result<usize> {
self.read_disk_at(0, true, id, buffer, offset)
}
pub(super) fn disk_size(&mut self, primary: usize, mapped: bool, id: &str) -> Result<u64> {
if mapped {
self.image_map_in(primary, id).map(|(_, size)| size)
} else {
self.volumes[primary].size(id)
}
}
pub(super) fn read_disk_at(
&mut self,
primary: usize,
mapped: bool,
id: &str,
buffer: &mut [u8],
offset: u64,
) -> Result<usize> {
let size = self.disk_size(primary, mapped, id)?;
if offset >= size || buffer.is_empty() {
return Ok(0);
}
let length = (size - offset).min(buffer.len() as u64) as usize;
if length >= LOGICAL_READ_AHEAD_BYTES {
let read =
self.read_disk_at_impl(primary, mapped, id, &mut buffer[..length], offset, false)?;
self.retain_complete_pages(primary, id, &buffer[..read], offset)?;
return Ok(read);
}
let mut done = 0;
while done < length {
let position = offset
.checked_add(done as u64)
.ok_or_else(|| malformed("AFF4 read offset overflow"))?;
let page_start =
position / LOGICAL_READ_AHEAD_BYTES as u64 * LOGICAL_READ_AHEAD_BYTES as u64;
let page_offset = usize::try_from(position - page_start)
.map_err(|_| malformed("AFF4 page offset overflow"))?;
let key = self.disk_page_key(primary, id, page_start)?;
let cached = self.volumes[primary]
.decoded_cache
.lock()
.map_err(|_| malformed("AFF4 decoded cache lock poisoned"))?
.get_page(&key);
let page = if let Some(cached) = cached {
Some(cached)
} else {
let page_length = LOGICAL_READ_AHEAD_BYTES
.min(usize::try_from(size - page_start).unwrap_or(LOGICAL_READ_AHEAD_BYTES));
let mut page = vec![0; page_length];
match self.read_disk_at_impl(primary, mapped, id, &mut page, page_start, false) {
Ok(read) if read == page_length => {
let page = Arc::new(page);
let mut cache = self.volumes[primary]
.decoded_cache
.lock()
.map_err(|_| malformed("AFF4 decoded cache lock poisoned"))?;
cache.record_read_ahead_prefetch();
cache.insert(key, Arc::clone(&page));
Some(page)
}
Ok(_) => return Err(malformed("truncated AFF4 disk page")),
Err(_) => None,
}
};
let Some(page) = page else {
let page_remaining = LOGICAL_READ_AHEAD_BYTES - page_offset;
let take = page_remaining.min(length - done);
let read = self.read_disk_at_impl(
primary,
mapped,
id,
&mut buffer[done..done + take],
position,
false,
)?;
if read != take {
return Err(malformed("truncated AFF4 disk read"));
}
done += take;
continue;
};
let take = (page.len() - page_offset).min(length - done);
buffer[done..done + take].copy_from_slice(&page[page_offset..page_offset + take]);
done += take;
}
Ok(length)
}
fn disk_page_key(&self, primary: usize, id: &str, offset: u64) -> Result<DecodedCacheKey> {
let volume = self
.volumes
.get(primary)
.ok_or_else(|| malformed("AFF4 primary volume index out of range"))?
.volume
.clone();
Ok(DecodedCacheKey::DiskPage {
volume,
resource: id.to_owned(),
offset,
})
}
fn retain_complete_pages(
&self,
primary: usize,
id: &str,
buffer: &[u8],
offset: u64,
) -> Result<()> {
let request_end = offset
.checked_add(buffer.len() as u64)
.ok_or_else(|| malformed("AFF4 read offset overflow"))?;
let page_bytes = LOGICAL_READ_AHEAD_BYTES as u64;
let mut page_start = match offset % page_bytes {
0 => offset,
remainder => offset
.checked_add(page_bytes - remainder)
.ok_or_else(|| malformed("AFF4 page offset overflow"))?,
};
while let Some(page_end) = page_start.checked_add(page_bytes) {
if page_end > request_end {
break;
}
let begin = usize::try_from(page_start - offset)
.map_err(|_| malformed("AFF4 page offset overflow"))?;
let page = Arc::new(buffer[begin..begin + LOGICAL_READ_AHEAD_BYTES].to_vec());
let key = self.disk_page_key(primary, id, page_start)?;
let mut cache = self.volumes[primary]
.decoded_cache
.lock()
.map_err(|_| malformed("AFF4 decoded cache lock poisoned"))?;
cache.record_read_ahead_prefetch();
cache.insert(key, page);
page_start = page_end;
}
Ok(())
}
fn read_disk_at_impl(
&mut self,
primary: usize,
mapped: bool,
id: &str,
buffer: &mut [u8],
offset: u64,
use_decoded_cache: bool,
) -> Result<usize> {
if !mapped {
self.clear_previous_cache(primary);
return self.volumes[primary].read_at_impl(id, buffer, offset, use_decoded_cache);
}
let (map, size) = self.image_map_in(primary, id)?;
if offset >= size {
return Ok(0);
}
let length = (size - offset).min(buffer.len() as u64) as usize;
let mut done = 0;
while done < length {
let position = offset + done as u64;
let next = map.ranges.partition_point(|r| r.end <= position);
let (target, start, end) = match map.ranges.get(next) {
Some(range) if range.start <= position => (
range.target.as_str(),
range.offset + position - range.start,
range.end,
),
next => (map.gap.as_str(), position, next.map_or(size, |r| r.start)),
};
let take = (end - position).min((length - done) as u64) as usize;
let owner = if is_symbolic(target) {
0
} else {
*self
.owners
.get(target)
.ok_or_else(|| malformed("missing target volume"))?
};
self.clear_previous_cache(owner);
self.volumes[owner].read_inner(
target,
&mut buffer[done..done + take],
start,
&mut Vec::new(),
use_decoded_cache,
)?;
done += take;
}
Ok(length)
}
fn clear_previous_cache(&mut self, owner: usize) {
if self.cached_owner != Some(owner) {
if let Some(previous) = self.cached_owner {
self.volumes[previous].cache = None;
self.volumes[previous].index_cache = None;
self.volumes[previous].maps.clear();
}
self.cached_owner = Some(owner);
}
}
pub fn verify_image(
&mut self,
id: &str,
expected_sha256: Option<&str>,
progress: impl FnMut(u64, u64) -> ControlFlow<()>,
) -> Result<SetDigest> {
self.scan_image(id, expected_sha256, false, progress)
.map(|(digest, _)| digest)
}
fn scan_image(
&mut self,
id: &str,
expected_sha256: Option<&str>,
all_hashes: bool,
mut progress: impl FnMut(u64, u64) -> ControlFlow<()>,
) -> Result<(SetDigest, Option<Verification>)> {
if expected_sha256
.is_some_and(|s| s.len() != 64 || !s.bytes().all(|b| b.is_ascii_hexdigit()))
{
return Err(malformed("invalid external SHA256"));
}
let size = self.size(id)?;
if size > self.volumes[0].limits.verification_bytes {
return Err(malformed("verification byte limit exceeded"));
}
let mut buffer = vec![0; 1024 * 1024];
let mut offset = 0;
let mut hash = Sha256::new();
let mut extra = all_hashes.then(|| {
(
Md5::new(),
Sha1::new(),
sha2::Sha512::new(),
blake2::Blake2b512::new(),
)
});
loop {
if progress(offset, size).is_break() {
return Err(Error::Aborted);
}
if offset == size {
break;
}
let read = self.read_disk_at_impl(0, true, id, &mut buffer, offset, false)?;
if read == 0 {
return Err(malformed("truncated volume set"));
}
hash.update(&buffer[..read]);
if let Some((md5, sha1, sha512, blake2b)) = &mut extra {
md5.update(&buffer[..read]);
sha1.update(&buffer[..read]);
sha512.update(&buffer[..read]);
blake2b.update(&buffer[..read]);
}
offset += read as u64;
}
let sha256 = hex(&hash.finalize());
let verification = extra.map(|(md5, sha1, sha512, blake2b)| Verification {
bytes_verified: size,
md5: hex(&md5.finalize()),
sha1: hex(&sha1.finalize()),
sha256: sha256.clone(),
sha512: hex(&sha512.finalize()),
blake2b: hex(&blake2b.finalize()),
references_match: None,
unsupported_hashes: Vec::new(),
});
Ok((
SetDigest {
image: id.into(),
bytes: size,
external_match: expected_sha256.map(|s| s.eq_ignore_ascii_case(&sha256)),
sha256,
sources: self.sources(),
},
verification,
))
}
}
pub(super) fn is_symbolic(id: &str) -> bool {
id == format!("{NS}Zero")
|| id.starts_with(&format!("{NS}SymbolicStream"))
|| id == format!("{NS}UnknownData")
|| id == format!("{NS}UnreadableData")
}
fn map_interval(map: &Map, position: u64, size: u64) -> (&str, u64, u64) {
let next = map.ranges.partition_point(|range| range.end <= position);
match map.ranges.get(next) {
Some(range) if range.start <= position => (
&range.target,
range.offset + position - range.start,
range.end,
),
next => (&map.gap, position, next.map_or(size, |range| range.start)),
}
}