1use super::*;
3use std::path::PathBuf;
4
5#[path = "set_verify.rs"]
6mod set_verify;
7pub use set_verify::{SetVerification, SetVolumeVerification};
8
9#[derive(Debug, Clone, serde::Serialize)]
11pub struct VolumeSource {
12 pub stream: String,
14 pub volume: String,
16 pub path: PathBuf,
18}
19
20#[derive(Debug, Clone, serde::Serialize)]
22pub struct SetDigest {
23 pub image: String,
25 pub bytes: u64,
27 pub sha256: String,
29 pub external_match: Option<bool>,
31 pub sources: Vec<VolumeSource>,
33}
34
35pub struct VolumeSet {
39 volumes: Vec<Container>,
40 paths: Vec<PathBuf>,
41 owners: BTreeMap<String, usize>,
42 validated: BTreeMap<(usize, String), (Arc<Map>, u64)>,
43 cached_owner: Option<usize>,
44}
45
46impl VolumeSet {
47 pub fn open(paths: &[PathBuf]) -> Result<Self> {
51 Self::open_with_limits(paths, Limits::default())
52 }
53
54 pub fn open_with_limits(paths: &[PathBuf], limits: Limits) -> Result<Self> {
58 if paths.is_empty() || paths.len() > 128 {
59 return Err(malformed("volume set must contain 1 through 128 paths"));
60 }
61 let mut volumes = Vec::new();
62 let mut ids = BTreeSet::new();
63 let mut remaining = limits.clone();
64 for path in paths {
65 let mut volume = Container::open_with_limits(path, remaining.clone())?;
66 remaining.metadata_bytes -= volume.usage.metadata_bytes;
67 remaining.triples -= volume.usage.triples;
68 remaining.directory_bytes -= volume.usage.directory_bytes;
69 remaining.archive_entries -= volume.usage.entries;
70 volume.limits = limits.clone();
71 if volume.version != (1, 0) {
72 return Err(Error::Unsupported(
73 "volume sets currently support physical AFF4 1.0".into(),
74 ));
75 }
76 if !ids.insert(volume.volume.clone()) {
77 return Err(malformed("duplicate volume identifier"));
78 }
79 volumes.push(volume);
80 }
81 Self::from_containers(volumes, paths.to_vec())
82 }
83
84 pub(super) fn from_containers(
85 mut volumes: Vec<Container>,
86 paths: Vec<PathBuf>,
87 ) -> Result<Self> {
88 let ids: BTreeSet<_> = volumes.iter().map(|volume| volume.volume.clone()).collect();
89 if ids.len() != volumes.len() {
90 return Err(malformed("duplicate volume identifier"));
91 }
92 let mut owners = BTreeMap::new();
93 for (index, volume) in volumes.iter().enumerate() {
94 for id in volume
95 .graph
96 .keys()
97 .filter(|id| volume.has_type(id, "ImageStream"))
98 {
99 if let Some(stored) = volume.value(id, "stored")?
100 && !ids.contains(&stored)
101 {
102 return Err(malformed(format!("missing source volume {stored}")));
103 }
104 let Ok(size) = volume.number(id, "size") else {
105 continue;
106 };
107 let path = volume.path(id)?;
108 let owns = if size == 0 {
109 volume.value(id, "stored")?.as_deref() == Some(&volume.volume)
110 } else {
111 volume
112 .archive
113 .index_for_name(&format!("{path}/00000000"))
114 .is_some()
115 };
116 if owns {
117 if volume
118 .value(id, "stored")?
119 .is_some_and(|v| v != volume.volume)
120 {
121 return Err(malformed("local stream conflicts with declared owner"));
122 }
123 if owners.insert(id.clone(), index).is_some() {
124 return Err(malformed(format!("ambiguous stream owner: {id}")));
125 }
126 }
127 }
128 }
129 for (id, &owner) in &owners {
132 for volume in &volumes {
133 if !volume.graph.contains_key(id) {
134 continue;
135 }
136 for key in [
137 "size",
138 "chunkSize",
139 "chunksInSegment",
140 "compressionMethod",
141 "stored",
142 ] {
143 if let Some(value) = volume.value(id, key)? {
144 let expected = if key == "stored" {
145 Some(volumes[owner].volume.clone())
146 } else {
147 volumes[owner].value(id, key)?
148 };
149 if expected.as_ref() != Some(&value) {
150 return Err(malformed(format!("conflicting {key} for {id}")));
151 }
152 }
153 }
154 }
155 }
156 let decoded_cache = Arc::new(Mutex::new(DecodedChunkCache::new()));
157 for volume in &mut volumes {
158 volume.decoded_cache = Arc::clone(&decoded_cache);
159 }
160 Ok(Self {
161 volumes,
162 paths,
163 owners,
164 validated: BTreeMap::new(),
165 cached_owner: None,
166 })
167 }
168
169 pub fn images(&self) -> Vec<String> {
171 self.volumes[0]
172 .graph
173 .keys()
174 .filter(|id| self.volumes[0].has_type(id, "DiskImage"))
175 .cloned()
176 .collect()
177 }
178
179 pub fn sources(&self) -> Vec<VolumeSource> {
181 self.owners
182 .iter()
183 .map(|(id, &n)| VolumeSource {
184 stream: id.clone(),
185 volume: self.volumes[n].volume.clone(),
186 path: self.paths[n].clone(),
187 })
188 .collect()
189 }
190
191 pub fn reader_statistics(&self) -> ReaderStatistics {
193 self.volumes[0].reader_statistics()
194 }
195
196 pub fn reader_cache_info(&self) -> ReaderCacheInfo {
198 self.volumes[0].reader_cache_info()
199 }
200
201 fn image_map(&mut self, id: &str) -> Result<(Arc<Map>, u64)> {
203 self.image_map_in(0, id)
204 }
205
206 fn image_map_in(&mut self, primary: usize, id: &str) -> Result<(Arc<Map>, u64)> {
207 if let Some(result) = self.validated.get(&(primary, id.to_owned())) {
208 return Ok(result.clone());
209 }
210 if !self.volumes[primary].has_type(id, "DiskImage") {
211 return Err(Error::Unsupported("select a primary DiskImage".into()));
212 }
213 let target = self.volumes[primary]
214 .value(id, "dataStream")?
215 .ok_or_else(|| malformed("image has no dataStream"))?;
216 self.volumes[primary].load_map(&target)?;
217 let map = self.volumes[primary].maps[&target].clone();
218 self.volumes[primary].maps.clear();
220 let size = self.volumes[primary].number(&target, "size")?;
221 if self.volumes[primary].value(id, "size")?.is_some()
222 && self.volumes[primary].number(id, "size")? != size
223 {
224 return Err(malformed("image/map size mismatch"));
225 }
226 let contiguous = self.volumes[primary].has_type(id, "ContiguousImage");
227 let mut end = 0;
228 for range in &map.ranges {
229 if contiguous && range.start != end {
230 return Err(malformed("gap in contiguous image"));
231 }
232 end = range.end;
233 if is_symbolic(&range.target) {
234 continue;
235 }
236 let owner = *self
237 .owners
238 .get(&range.target)
239 .ok_or_else(|| malformed(format!("missing owner for {}", range.target)))?;
240 let size = self.volumes[owner].number(&range.target, "size")?;
241 if range
242 .offset
243 .checked_add(range.end - range.start)
244 .is_none_or(|end| end > size)
245 {
246 return Err(malformed("map exceeds owned stream"));
247 }
248 }
249 if contiguous && end != size {
250 return Err(malformed("truncated contiguous map"));
251 }
252 if !is_symbolic(&map.gap) && !self.owners.contains_key(&map.gap) {
253 return Err(Error::Unsupported("external/nested gap stream".into()));
254 }
255 self.validated.clear();
256 self.validated
257 .insert((primary, id.into()), (map.clone(), size));
258 Ok((map, size))
259 }
260
261 pub(super) fn equivalent_disk_maps(
262 &mut self,
263 left: usize,
264 right: usize,
265 id: &str,
266 ) -> Result<bool> {
267 let (left, size) = self.image_map_in(left, id)?;
268 let (right, other_size) = self.image_map_in(right, id)?;
269 if size != other_size {
270 return Ok(false);
271 }
272 let mut position = 0;
273 while position < size {
274 let (a, a_offset, a_end) = map_interval(&left, position, size);
275 let (b, b_offset, b_end) = map_interval(&right, position, size);
276 if a != b || (!is_symbolic(a) && a_offset != b_offset) {
277 return Ok(false);
278 }
279 position = a_end.min(b_end);
280 }
281 Ok(true)
282 }
283
284 pub fn size(&mut self, id: &str) -> Result<u64> {
286 self.image_map(id).map(|(_, size)| size)
287 }
288
289 pub fn read_at(&mut self, id: &str, buffer: &mut [u8], offset: u64) -> Result<usize> {
292 self.read_disk_at(0, true, id, buffer, offset)
293 }
294
295 pub(super) fn disk_size(&mut self, primary: usize, mapped: bool, id: &str) -> Result<u64> {
296 if mapped {
297 self.image_map_in(primary, id).map(|(_, size)| size)
298 } else {
299 self.volumes[primary].size(id)
300 }
301 }
302
303 pub(super) fn read_disk_at(
304 &mut self,
305 primary: usize,
306 mapped: bool,
307 id: &str,
308 buffer: &mut [u8],
309 offset: u64,
310 ) -> Result<usize> {
311 let size = self.disk_size(primary, mapped, id)?;
312 if offset >= size || buffer.is_empty() {
313 return Ok(0);
314 }
315 let length = (size - offset).min(buffer.len() as u64) as usize;
316 if length >= LOGICAL_READ_AHEAD_BYTES {
317 let read =
318 self.read_disk_at_impl(primary, mapped, id, &mut buffer[..length], offset, false)?;
319 self.retain_complete_pages(primary, id, &buffer[..read], offset)?;
320 return Ok(read);
321 }
322
323 let mut done = 0;
324 while done < length {
325 let position = offset
326 .checked_add(done as u64)
327 .ok_or_else(|| malformed("AFF4 read offset overflow"))?;
328 let page_start =
329 position / LOGICAL_READ_AHEAD_BYTES as u64 * LOGICAL_READ_AHEAD_BYTES as u64;
330 let page_offset = usize::try_from(position - page_start)
331 .map_err(|_| malformed("AFF4 page offset overflow"))?;
332 let key = self.disk_page_key(primary, id, page_start)?;
333 let cached = self.volumes[primary]
334 .decoded_cache
335 .lock()
336 .map_err(|_| malformed("AFF4 decoded cache lock poisoned"))?
337 .get_page(&key);
338 let page = if let Some(cached) = cached {
339 Some(cached)
340 } else {
341 let page_length = LOGICAL_READ_AHEAD_BYTES
342 .min(usize::try_from(size - page_start).unwrap_or(LOGICAL_READ_AHEAD_BYTES));
343 let mut page = vec![0; page_length];
344 match self.read_disk_at_impl(primary, mapped, id, &mut page, page_start, false) {
345 Ok(read) if read == page_length => {
346 let page = Arc::new(page);
347 let mut cache = self.volumes[primary]
348 .decoded_cache
349 .lock()
350 .map_err(|_| malformed("AFF4 decoded cache lock poisoned"))?;
351 cache.record_read_ahead_prefetch();
352 cache.insert(key, Arc::clone(&page));
353 Some(page)
354 }
355 Ok(_) => return Err(malformed("truncated AFF4 disk page")),
356 Err(_) => None,
357 }
358 };
359 let Some(page) = page else {
360 let page_remaining = LOGICAL_READ_AHEAD_BYTES - page_offset;
363 let take = page_remaining.min(length - done);
364 let read = self.read_disk_at_impl(
365 primary,
366 mapped,
367 id,
368 &mut buffer[done..done + take],
369 position,
370 false,
371 )?;
372 if read != take {
373 return Err(malformed("truncated AFF4 disk read"));
374 }
375 done += take;
376 continue;
377 };
378 let take = (page.len() - page_offset).min(length - done);
379 buffer[done..done + take].copy_from_slice(&page[page_offset..page_offset + take]);
380 done += take;
381 }
382 Ok(length)
383 }
384
385 fn disk_page_key(&self, primary: usize, id: &str, offset: u64) -> Result<DecodedCacheKey> {
386 let volume = self
387 .volumes
388 .get(primary)
389 .ok_or_else(|| malformed("AFF4 primary volume index out of range"))?
390 .volume
391 .clone();
392 Ok(DecodedCacheKey::DiskPage {
393 volume,
394 resource: id.to_owned(),
395 offset,
396 })
397 }
398
399 fn retain_complete_pages(
400 &self,
401 primary: usize,
402 id: &str,
403 buffer: &[u8],
404 offset: u64,
405 ) -> Result<()> {
406 let request_end = offset
407 .checked_add(buffer.len() as u64)
408 .ok_or_else(|| malformed("AFF4 read offset overflow"))?;
409 let page_bytes = LOGICAL_READ_AHEAD_BYTES as u64;
410 let mut page_start = match offset % page_bytes {
411 0 => offset,
412 remainder => offset
413 .checked_add(page_bytes - remainder)
414 .ok_or_else(|| malformed("AFF4 page offset overflow"))?,
415 };
416 while let Some(page_end) = page_start.checked_add(page_bytes) {
417 if page_end > request_end {
418 break;
419 }
420 let begin = usize::try_from(page_start - offset)
421 .map_err(|_| malformed("AFF4 page offset overflow"))?;
422 let page = Arc::new(buffer[begin..begin + LOGICAL_READ_AHEAD_BYTES].to_vec());
423 let key = self.disk_page_key(primary, id, page_start)?;
424 let mut cache = self.volumes[primary]
425 .decoded_cache
426 .lock()
427 .map_err(|_| malformed("AFF4 decoded cache lock poisoned"))?;
428 cache.record_read_ahead_prefetch();
429 cache.insert(key, page);
430 page_start = page_end;
431 }
432 Ok(())
433 }
434
435 fn read_disk_at_impl(
436 &mut self,
437 primary: usize,
438 mapped: bool,
439 id: &str,
440 buffer: &mut [u8],
441 offset: u64,
442 use_decoded_cache: bool,
443 ) -> Result<usize> {
444 if !mapped {
445 self.clear_previous_cache(primary);
446 return self.volumes[primary].read_at_impl(id, buffer, offset, use_decoded_cache);
447 }
448 let (map, size) = self.image_map_in(primary, id)?;
449 if offset >= size {
450 return Ok(0);
451 }
452 let length = (size - offset).min(buffer.len() as u64) as usize;
453 let mut done = 0;
454 while done < length {
455 let position = offset + done as u64;
456 let next = map.ranges.partition_point(|r| r.end <= position);
457 let (target, start, end) = match map.ranges.get(next) {
458 Some(range) if range.start <= position => (
459 range.target.as_str(),
460 range.offset + position - range.start,
461 range.end,
462 ),
463 next => (map.gap.as_str(), position, next.map_or(size, |r| r.start)),
464 };
465 let take = (end - position).min((length - done) as u64) as usize;
466 let owner = if is_symbolic(target) {
467 0
468 } else {
469 *self
470 .owners
471 .get(target)
472 .ok_or_else(|| malformed("missing target volume"))?
473 };
474 self.clear_previous_cache(owner);
475 self.volumes[owner].read_inner(
476 target,
477 &mut buffer[done..done + take],
478 start,
479 &mut Vec::new(),
480 use_decoded_cache,
481 )?;
482 done += take;
483 }
484 Ok(length)
485 }
486
487 fn clear_previous_cache(&mut self, owner: usize) {
488 if self.cached_owner != Some(owner) {
489 if let Some(previous) = self.cached_owner {
490 self.volumes[previous].cache = None;
491 self.volumes[previous].index_cache = None;
492 self.volumes[previous].maps.clear();
493 }
494 self.cached_owner = Some(owner);
495 }
496 }
497
498 pub fn verify_image(
502 &mut self,
503 id: &str,
504 expected_sha256: Option<&str>,
505 progress: impl FnMut(u64, u64) -> ControlFlow<()>,
506 ) -> Result<SetDigest> {
507 self.scan_image(id, expected_sha256, false, progress)
508 .map(|(digest, _)| digest)
509 }
510
511 fn scan_image(
512 &mut self,
513 id: &str,
514 expected_sha256: Option<&str>,
515 all_hashes: bool,
516 mut progress: impl FnMut(u64, u64) -> ControlFlow<()>,
517 ) -> Result<(SetDigest, Option<Verification>)> {
518 if expected_sha256
519 .is_some_and(|s| s.len() != 64 || !s.bytes().all(|b| b.is_ascii_hexdigit()))
520 {
521 return Err(malformed("invalid external SHA256"));
522 }
523 let size = self.size(id)?;
524 if size > self.volumes[0].limits.verification_bytes {
525 return Err(malformed("verification byte limit exceeded"));
526 }
527 let mut buffer = vec![0; 1024 * 1024];
528 let mut offset = 0;
529 let mut hash = Sha256::new();
530 let mut extra = all_hashes.then(|| {
531 (
532 Md5::new(),
533 Sha1::new(),
534 sha2::Sha512::new(),
535 blake2::Blake2b512::new(),
536 )
537 });
538 loop {
539 if progress(offset, size).is_break() {
540 return Err(Error::Aborted);
541 }
542 if offset == size {
543 break;
544 }
545 let read = self.read_disk_at_impl(0, true, id, &mut buffer, offset, false)?;
546 if read == 0 {
547 return Err(malformed("truncated volume set"));
548 }
549 hash.update(&buffer[..read]);
550 if let Some((md5, sha1, sha512, blake2b)) = &mut extra {
551 md5.update(&buffer[..read]);
552 sha1.update(&buffer[..read]);
553 sha512.update(&buffer[..read]);
554 blake2b.update(&buffer[..read]);
555 }
556 offset += read as u64;
557 }
558 let sha256 = hex(&hash.finalize());
559 let verification = extra.map(|(md5, sha1, sha512, blake2b)| Verification {
560 bytes_verified: size,
561 md5: hex(&md5.finalize()),
562 sha1: hex(&sha1.finalize()),
563 sha256: sha256.clone(),
564 sha512: hex(&sha512.finalize()),
565 blake2b: hex(&blake2b.finalize()),
566 references_match: None,
567 unsupported_hashes: Vec::new(),
568 });
569 Ok((
570 SetDigest {
571 image: id.into(),
572 bytes: size,
573 external_match: expected_sha256.map(|s| s.eq_ignore_ascii_case(&sha256)),
574 sha256,
575 sources: self.sources(),
576 },
577 verification,
578 ))
579 }
580}
581
582pub(super) fn is_symbolic(id: &str) -> bool {
583 id == format!("{NS}Zero")
584 || id.starts_with(&format!("{NS}SymbolicStream"))
585 || id == format!("{NS}UnknownData")
586 || id == format!("{NS}UnreadableData")
587}
588
589fn map_interval(map: &Map, position: u64, size: u64) -> (&str, u64, u64) {
590 let next = map.ranges.partition_point(|range| range.end <= position);
591 match map.ranges.get(next) {
592 Some(range) if range.start <= position => (
593 &range.target,
594 range.offset + position - range.start,
595 range.end,
596 ),
597 next => (&map.gap, position, next.map_or(size, |range| range.start)),
598 }
599}