1use std::collections::{BTreeMap, BTreeSet};
2use std::fs::File;
3use std::io::{Read, Write};
4use std::ops::ControlFlow;
5use std::path::Path;
6use std::sync::{Arc, Mutex};
7use std::time::{Duration, Instant};
8
9use lru::LruCache;
10use md5::{Digest, Md5};
11use oxrdf::{NamedOrBlankNode, Term};
12use oxttl::TurtleParser;
13use sha1::Sha1;
14use sha2::Sha256;
15use zip::{CompressionMethod, ZipArchive};
16
17use crate::{Error, Result, malformed};
18use base64::Engine;
19
20#[path = "integrity.rs"]
21mod integrity;
22pub use integrity::{CheckOutcome, IntegrityCheck, MetadataVerification};
23
24#[path = "verify_all.rs"]
25mod verify_all;
26pub use verify_all::{ByteCoverage, ContainerVerification, ResourceVerification};
27
28#[path = "scan.rs"]
29mod scan;
30pub use scan::MetadataScan;
31
32#[path = "archive.rs"]
33mod archive;
34
35#[path = "volume_set.rs"]
36mod volume_set;
37pub use volume_set::{SetDigest, SetVerification, SetVolumeVerification, VolumeSet, VolumeSource};
38
39#[path = "discovery.rs"]
40mod discovery;
41pub use discovery::{BackingVolume, DiskImageSet, PhysicalDisk, PhysicalDiskReader};
42
43const NS: &str = "http://aff4.org/Schema#";
44const RDF_TYPE: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type";
45const LOGICAL_NS: &str = "https://aff4.org/Schema/2022/#";
46const LEGACY_LOGICAL_NS: &str = "http://aff4.org/Schema/2022/#";
47const BASE64: &str = "http://www.w3.org/2001/XMLSchema#base64Binary";
48const DECODED_CACHE_CAPACITY_BYTES: usize = 128 * 1024 * 1024;
49const LOGICAL_READ_AHEAD_BYTES: usize = 1024 * 1024;
50
51#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize)]
52#[non_exhaustive]
53pub struct ReaderStatistics {
58 decoded_cache_hits: u64,
59 decoded_cache_misses: u64,
60 read_ahead_cache_hits: u64,
61 read_ahead_cache_misses: u64,
62 read_ahead_prefetches: u64,
63 decoded_cache_evictions: u64,
64 decoded_bytes: u64,
65 decode_nanos: u64,
66 stored_member_range_reads: u64,
67 stored_member_range_bytes: u64,
68}
69
70impl ReaderStatistics {
71 pub fn decoded_cache_hits(&self) -> u64 {
73 self.decoded_cache_hits
74 }
75
76 pub fn decoded_cache_misses(&self) -> u64 {
78 self.decoded_cache_misses
79 }
80
81 pub fn read_ahead_cache_hits(&self) -> u64 {
83 self.read_ahead_cache_hits
84 }
85
86 pub fn read_ahead_cache_misses(&self) -> u64 {
88 self.read_ahead_cache_misses
89 }
90
91 pub fn read_ahead_prefetches(&self) -> u64 {
93 self.read_ahead_prefetches
94 }
95
96 pub fn decoded_cache_evictions(&self) -> u64 {
98 self.decoded_cache_evictions
99 }
100
101 pub fn decoded_bytes(&self) -> u64 {
103 self.decoded_bytes
104 }
105
106 pub fn decode_nanos(&self) -> u64 {
108 self.decode_nanos
109 }
110
111 pub fn stored_member_range_reads(&self) -> u64 {
113 self.stored_member_range_reads
114 }
115
116 pub fn stored_member_range_bytes(&self) -> u64 {
118 self.stored_member_range_bytes
119 }
120
121 #[must_use]
123 pub fn saturating_delta(self, earlier: Self) -> Self {
124 Self {
125 decoded_cache_hits: self
126 .decoded_cache_hits
127 .saturating_sub(earlier.decoded_cache_hits),
128 decoded_cache_misses: self
129 .decoded_cache_misses
130 .saturating_sub(earlier.decoded_cache_misses),
131 read_ahead_cache_hits: self
132 .read_ahead_cache_hits
133 .saturating_sub(earlier.read_ahead_cache_hits),
134 read_ahead_cache_misses: self
135 .read_ahead_cache_misses
136 .saturating_sub(earlier.read_ahead_cache_misses),
137 read_ahead_prefetches: self
138 .read_ahead_prefetches
139 .saturating_sub(earlier.read_ahead_prefetches),
140 decoded_cache_evictions: self
141 .decoded_cache_evictions
142 .saturating_sub(earlier.decoded_cache_evictions),
143 decoded_bytes: self.decoded_bytes.saturating_sub(earlier.decoded_bytes),
144 decode_nanos: self.decode_nanos.saturating_sub(earlier.decode_nanos),
145 stored_member_range_reads: self
146 .stored_member_range_reads
147 .saturating_sub(earlier.stored_member_range_reads),
148 stored_member_range_bytes: self
149 .stored_member_range_bytes
150 .saturating_sub(earlier.stored_member_range_bytes),
151 }
152 }
153}
154
155#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize)]
156#[non_exhaustive]
157pub struct ReaderCacheInfo {
159 capacity_bytes: u64,
160 current_bytes: u64,
161 peak_bytes: u64,
162 entries: u64,
163}
164
165impl ReaderCacheInfo {
166 pub fn capacity_bytes(&self) -> u64 {
168 self.capacity_bytes
169 }
170
171 pub fn current_bytes(&self) -> u64 {
173 self.current_bytes
174 }
175
176 pub fn peak_bytes(&self) -> u64 {
178 self.peak_bytes
179 }
180
181 pub fn entries(&self) -> u64 {
183 self.entries
184 }
185}
186
187#[derive(Clone, Hash, PartialEq, Eq)]
188enum DecodedCacheKey {
189 Chunk {
190 volume: String,
191 resource: String,
192 chunk: u64,
193 },
194 DiskPage {
195 volume: String,
196 resource: String,
197 offset: u64,
198 },
199}
200
201struct DecodedChunkCache {
202 entries: LruCache<DecodedCacheKey, Arc<Vec<u8>>>,
203 capacity_bytes: usize,
204 current_bytes: usize,
205 peak_bytes: usize,
206 statistics: ReaderStatistics,
207}
208
209impl DecodedChunkCache {
210 fn new() -> Self {
211 Self::with_capacity(DECODED_CACHE_CAPACITY_BYTES)
212 }
213
214 fn with_capacity(capacity_bytes: usize) -> Self {
215 Self {
216 entries: LruCache::unbounded(),
217 capacity_bytes,
218 current_bytes: 0,
219 peak_bytes: 0,
220 statistics: ReaderStatistics::default(),
221 }
222 }
223
224 fn get_chunk(&mut self, key: &DecodedCacheKey) -> Option<Arc<Vec<u8>>> {
225 let result = self.entries.get(key).cloned();
226 if result.is_some() {
227 self.statistics.decoded_cache_hits =
228 self.statistics.decoded_cache_hits.saturating_add(1);
229 } else {
230 self.statistics.decoded_cache_misses =
231 self.statistics.decoded_cache_misses.saturating_add(1);
232 }
233 result
234 }
235
236 fn get_page(&mut self, key: &DecodedCacheKey) -> Option<Arc<Vec<u8>>> {
237 let result = self.entries.get(key).cloned();
238 if result.is_some() {
239 self.statistics.read_ahead_cache_hits =
240 self.statistics.read_ahead_cache_hits.saturating_add(1);
241 } else {
242 self.statistics.read_ahead_cache_misses =
243 self.statistics.read_ahead_cache_misses.saturating_add(1);
244 }
245 result
246 }
247
248 fn insert(&mut self, key: DecodedCacheKey, decoded: Arc<Vec<u8>>) {
249 let bytes = decoded.len();
250 if bytes > self.capacity_bytes {
251 return;
252 }
253 if let Some(previous) = self.entries.put(key, decoded) {
254 self.current_bytes = self.current_bytes.saturating_sub(previous.len());
255 }
256 self.current_bytes = self.current_bytes.saturating_add(bytes);
257 while self.current_bytes > self.capacity_bytes {
258 let Some((_, evicted)) = self.entries.pop_lru() else {
259 break;
260 };
261 self.current_bytes = self.current_bytes.saturating_sub(evicted.len());
262 self.statistics.decoded_cache_evictions =
263 self.statistics.decoded_cache_evictions.saturating_add(1);
264 }
265 self.peak_bytes = self.peak_bytes.max(self.current_bytes);
266 }
267
268 fn record_decode(&mut self, bytes: usize, elapsed: Duration) {
269 self.statistics.decoded_bytes = self
270 .statistics
271 .decoded_bytes
272 .saturating_add(u64::try_from(bytes).unwrap_or(u64::MAX));
273 self.statistics.decode_nanos = self
274 .statistics
275 .decode_nanos
276 .saturating_add(u64::try_from(elapsed.as_nanos()).unwrap_or(u64::MAX));
277 }
278
279 fn record_read_ahead_prefetch(&mut self) {
280 self.statistics.read_ahead_prefetches =
281 self.statistics.read_ahead_prefetches.saturating_add(1);
282 }
283
284 fn record_stored_member_range_read(&mut self, bytes: usize) {
285 self.statistics.stored_member_range_reads =
286 self.statistics.stored_member_range_reads.saturating_add(1);
287 self.statistics.stored_member_range_bytes = self
288 .statistics
289 .stored_member_range_bytes
290 .saturating_add(u64::try_from(bytes).unwrap_or(u64::MAX));
291 }
292
293 fn clear(&mut self) {
294 self.entries.clear();
295 self.current_bytes = 0;
296 }
297
298 fn info(&self) -> ReaderCacheInfo {
299 ReaderCacheInfo {
300 capacity_bytes: self.capacity_bytes as u64,
301 current_bytes: self.current_bytes as u64,
302 peak_bytes: self.peak_bytes as u64,
303 entries: self.entries.len() as u64,
304 }
305 }
306}
307
308#[derive(Debug, Clone, serde::Serialize)]
310pub struct Limits {
311 pub directory_bytes: u64,
313 pub archive_entries: usize,
315 pub metadata_bytes: u64,
317 pub member_bytes: u64,
319 pub chunk_bytes: u64,
321 pub triples: usize,
323 pub map_bytes: usize,
325 pub verification_bytes: u64,
328}
329impl Default for Limits {
330 fn default() -> Self {
331 Self {
332 directory_bytes: 64 * 1024 * 1024,
333 archive_entries: 200_000,
334 metadata_bytes: 16 * 1024 * 1024,
335 member_bytes: 128 * 1024 * 1024,
336 chunk_bytes: 16 * 1024 * 1024,
337 triples: 200_000,
338 map_bytes: 128 * 1024 * 1024,
339 verification_bytes: 64 * 1024 * 1024 * 1024 * 1024,
340 }
341 }
342}
343
344impl Limits {
345 pub const fn unrestricted() -> Self {
352 Self {
353 directory_bytes: isize::MAX as u64,
354 archive_entries: isize::MAX as usize,
355 metadata_bytes: isize::MAX as u64,
356 member_bytes: isize::MAX as u64,
357 chunk_bytes: isize::MAX as u64,
358 triples: isize::MAX as usize,
359 map_bytes: isize::MAX as usize,
360 verification_bytes: u64::MAX,
361 }
362 }
363}
364
365#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
367pub struct Property {
368 pub predicate: String,
370 pub value: String,
372 pub datatype: Option<String>,
374 pub language: Option<String>,
376}
377
378#[derive(Debug, Clone, serde::Serialize)]
380pub struct StreamInfo {
381 pub id: String,
383 pub types: Vec<String>,
385 pub size: Option<u64>,
387}
388
389#[derive(Debug, Clone, serde::Serialize)]
391pub struct Verification {
392 pub bytes_verified: u64,
394 pub md5: String,
396 pub sha1: String,
398 pub sha256: String,
400 pub sha512: String,
402 pub blake2b: String,
404 pub references_match: Option<bool>,
406 pub unsupported_hashes: Vec<String>,
408}
409
410pub struct Container {
413 archive: ZipArchive<File>,
414 stored_source: File,
415 volume: String,
416 graph: BTreeMap<String, Vec<Property>>,
417 limits: Limits,
418 cache: Option<(String, Vec<u8>)>,
419 index_cache: Option<(String, Vec<u8>)>,
420 stored_member: Option<StoredMember>,
421 decoded_cache: Arc<Mutex<DecodedChunkCache>>,
422 maps: BTreeMap<String, Arc<Map>>,
423 version: (u32, u32),
424 metadata_members: BTreeMap<String, String>,
425 usage: archive::Usage,
426}
427
428struct StoredMember {
429 name: String,
430 size: u64,
431 data_start: Option<u64>,
432}
433
434struct PositionedReader<'a> {
435 container: &'a mut Container,
436 id: String,
437 position: u64,
438}
439
440impl Read for PositionedReader<'_> {
441 fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
442 let count = self
443 .container
444 .read_at(&self.id, buffer, self.position)
445 .map_err(std::io::Error::other)?;
446 self.position += count as u64;
447 Ok(count)
448 }
449}
450
451#[derive(Clone)]
452struct Range {
453 start: u64,
454 end: u64,
455 offset: u64,
456 target: String,
457}
458#[derive(Clone)]
459struct Map {
460 ranges: Vec<Range>,
461 gap: String,
462}
463
464impl Container {
465 pub fn open(path: impl AsRef<Path>) -> Result<Self> {
467 Self::open_with_limits(path, Limits::default())
468 }
469
470 pub fn open_with_limits(path: impl AsRef<Path>, limits: Limits) -> Result<Self> {
473 let (mut archive, mut usage) = archive::open(path.as_ref(), &limits)?;
474 let stored_source = File::open(path.as_ref())?;
475 let mut names = BTreeSet::new();
476 for name in archive.file_names() {
477 if !names.insert(name.to_owned()) {
478 return Err(malformed("duplicate ZIP member name"));
479 }
480 }
481 let version = member(&mut archive, "version.txt", 4096)?;
482 let version =
483 std::str::from_utf8(&version).map_err(|_| malformed("invalid version text"))?;
484 let mut fields = BTreeMap::new();
485 for line in version.lines() {
486 if let Some((key, value)) = line.split_once('=')
487 && fields.insert(key.trim(), value.trim()).is_some()
488 {
489 return Err(malformed("duplicate version field"));
490 }
491 }
492 let version = (
493 fields
494 .get("major")
495 .and_then(|n| n.parse().ok())
496 .ok_or_else(|| malformed("missing or invalid major version"))?,
497 fields
498 .get("minor")
499 .and_then(|n| n.parse().ok())
500 .ok_or_else(|| malformed("missing or invalid minor version"))?,
501 );
502 if !matches!(version, (1, 0) | (1, 1) | (2, 1)) {
503 return Err(Error::Unsupported(
504 "container version (supported: 1.0, legacy logical 1.1, draft logical 2.1)".into(),
505 ));
506 }
507 let comment = std::str::from_utf8(archive.comment())
508 .map_err(|_| malformed("volume comment is not UTF-8"))?
509 .trim_end_matches('\0')
510 .trim()
511 .to_owned();
512 let description = match archive.index_for_name("container.description") {
513 Some(_) => String::from_utf8(member(&mut archive, "container.description", 4096)?)
514 .map_err(|_| malformed("volume name is not UTF-8"))?,
515 None => comment.clone(),
516 };
517 let volume = description.trim_end_matches('\0').trim().to_owned();
518 if !volume.starts_with("aff4://") || volume.len() <= 7 {
519 return Err(malformed("missing AFF4 volume URI"));
520 }
521 if !comment.is_empty() && comment != volume {
522 return Err(malformed("conflicting volume identifiers"));
523 }
524 let mut graph: BTreeMap<String, Vec<Property>> = BTreeMap::new();
525 let mut pending = vec!["information.turtle".to_owned()];
526 let mut seen = BTreeSet::new();
527 let mut metadata_members = BTreeMap::new();
528 let mut remaining = limits.metadata_bytes;
529 let mut count = 0;
530 while let Some(name) = pending.pop() {
531 if !seen.insert(name.clone()) {
532 continue;
533 }
534 let turtle = member(&mut archive, &name, remaining)?;
535 remaining -= turtle.len() as u64;
536 metadata_members.insert(name.clone(), hex(&Sha256::digest(&turtle)));
537 for triple in TurtleParser::new().for_reader(turtle.as_slice()) {
538 if count >= limits.triples {
539 return Err(malformed("RDF triple limit exceeded"));
540 }
541 let triple = triple.map_err(|error| malformed(format!("Turtle: {error}")))?;
542 count += 1;
543 let subject = match triple.subject {
544 NamedOrBlankNode::NamedNode(n) => n.into_string(),
545 NamedOrBlankNode::BlankNode(n) => format!("_:{n}"),
546 };
547 let (value, datatype, language) = match triple.object {
548 Term::NamedNode(n) => (n.into_string(), None, None),
549 Term::BlankNode(n) => (format!("_:{n}"), None, None),
550 Term::Literal(l) => (
551 l.value().to_owned(),
552 Some(l.datatype().as_str().to_owned()),
553 l.language().map(str::to_owned),
554 ),
555 };
556 if is_property(triple.predicate.as_str(), "imports") {
557 if datatype.is_some() {
558 return Err(malformed("metadata import must be an IRI"));
559 }
560 pending.push(storage_name(&archive, &volume, &value, version)?);
561 }
562 graph.entry(subject).or_default().push(Property {
563 predicate: triple.predicate.into_string(),
564 value,
565 datatype,
566 language,
567 });
568 }
569 }
570 usage.metadata_bytes = limits.metadata_bytes - remaining;
571 usage.triples = count;
572 Ok(Self {
573 archive,
574 stored_source,
575 volume,
576 graph,
577 limits,
578 cache: None,
579 index_cache: None,
580 stored_member: None,
581 decoded_cache: Arc::new(Mutex::new(DecodedChunkCache::new())),
582 maps: BTreeMap::new(),
583 version,
584 metadata_members,
585 usage,
586 })
587 }
588
589 pub fn metadata(&self) -> &BTreeMap<String, Vec<Property>> {
591 &self.graph
592 }
593
594 pub fn volume_id(&self) -> &str {
596 &self.volume
597 }
598
599 pub fn version(&self) -> (u32, u32) {
601 self.version
602 }
603
604 pub fn reader_statistics(&self) -> ReaderStatistics {
606 self.decoded_cache
607 .lock()
608 .unwrap_or_else(std::sync::PoisonError::into_inner)
609 .statistics
610 }
611
612 pub fn reader_cache_info(&self) -> ReaderCacheInfo {
614 self.decoded_cache
615 .lock()
616 .unwrap_or_else(std::sync::PoisonError::into_inner)
617 .info()
618 }
619
620 pub fn streams(&self) -> Result<Vec<StreamInfo>> {
622 self.graph
623 .keys()
624 .filter(|id| {
625 [
626 "Image",
627 "DiskImage",
628 "ImageStream",
629 "Map",
630 "FileImage",
631 "FileSubStream",
632 ]
633 .iter()
634 .any(|t| self.has_type(id, t))
635 })
636 .map(|id| {
637 Ok(StreamInfo {
638 id: id.clone(),
639 types: self
640 .properties(id)
641 .filter(|p| p.predicate == RDF_TYPE)
642 .map(|p| p.value.clone())
643 .collect(),
644 size: self.stream_size(id, &mut Vec::new()).ok(),
645 })
646 })
647 .collect()
648 }
649
650 pub fn size(&self, id: &str) -> Result<u64> {
652 self.stream_size(id, &mut Vec::new())
653 }
654
655 pub fn read_at(&mut self, id: &str, buffer: &mut [u8], offset: u64) -> Result<usize> {
658 self.read_at_impl(id, buffer, offset, true)
659 }
660
661 pub fn sequential_reader(&mut self, id: &str) -> Result<Box<dyn Read + '_>> {
666 let size = self.size(id)?;
667 self.read_inner(id, &mut [], 0, &mut Vec::new(), false)?;
668 let target = self.linear_backing(id)?;
669 if self.has_type(&target, "ZipSegment") || self.has_type(&target, "zip_segment") {
670 let path = self.path(&target)?;
671 let file = self.archive.by_name(&path)?;
672 if file.size() != size {
673 return Err(malformed("ZIP segment size mismatch"));
674 }
675 return Ok(Box::new(file));
676 }
677 Ok(Box::new(PositionedReader {
678 container: self,
679 id: id.to_owned(),
680 position: 0,
681 }))
682 }
683
684 fn read_at_uncached(&mut self, id: &str, buffer: &mut [u8], offset: u64) -> Result<usize> {
685 self.read_at_impl(id, buffer, offset, false)
686 }
687
688 fn read_at_impl(
689 &mut self,
690 id: &str,
691 buffer: &mut [u8],
692 offset: u64,
693 use_decoded_cache: bool,
694 ) -> Result<usize> {
695 let size = self.size(id)?;
696 if offset >= size || buffer.is_empty() {
697 return Ok(0);
698 }
699 let length = (size - offset).min(buffer.len() as u64) as usize;
700 self.read_inner(
701 id,
702 &mut buffer[..length],
703 offset,
704 &mut Vec::new(),
705 use_decoded_cache,
706 )?;
707 Ok(length)
708 }
709
710 pub fn verify(
713 &mut self,
714 id: &str,
715 progress: impl FnMut(u64, u64) -> ControlFlow<()>,
716 ) -> Result<Verification> {
717 self.copy_verified(id, &mut std::io::sink(), progress)
718 }
719
720 pub fn copy_verified(
724 &mut self,
725 id: &str,
726 output: &mut impl Write,
727 progress: impl FnMut(u64, u64) -> ControlFlow<()>,
728 ) -> Result<Verification> {
729 self.copy_verified_impl(id, output, None, progress)
730 .map(|(verification, _)| verification)
731 }
732
733 fn verify_with_paired_blocks(
734 &mut self,
735 id: &str,
736 block_size: usize,
737 progress: impl FnMut(u64, u64) -> ControlFlow<()>,
738 ) -> Result<(Verification, PairedBlockDigests)> {
739 let (verification, blocks) =
740 self.copy_verified_impl(id, &mut std::io::sink(), Some(block_size), progress)?;
741 Ok((
742 verification,
743 blocks.expect("paired block hashing requested"),
744 ))
745 }
746
747 fn copy_verified_impl(
748 &mut self,
749 id: &str,
750 output: &mut impl Write,
751 block_size: Option<usize>,
752 mut progress: impl FnMut(u64, u64) -> ControlFlow<()>,
753 ) -> Result<(Verification, Option<PairedBlockDigests>)> {
754 let size = self.size(id)?;
755 if size > self.limits.verification_bytes {
756 return Err(malformed("verification byte limit exceeded"));
757 }
758 self.cache = None;
759 self.index_cache = None;
760 self.decoded_cache
761 .lock()
762 .map_err(|_| malformed("AFF4 decoded cache lock poisoned"))?
763 .clear();
764 self.maps.clear();
765 self.read_inner(id, &mut [], 0, &mut Vec::new(), false)?;
766 let mut md5 = Md5::new();
767 let mut sha1 = Sha1::new();
768 let mut sha256 = Sha256::new();
769 let mut sha512 = sha2::Sha512::new();
770 let mut blake2b = <blake2::Blake2b512 as blake2::Digest>::new();
771 let mut blocks = block_size.map(PairedBlockHasher::new);
772 let mut write_error = None;
773 let walked = self.walk_bytes(id, |bytes, done| {
774 if let Err(error) = output.write_all(bytes) {
775 write_error = Some(error);
776 return ControlFlow::Break(());
777 }
778 md5.update(bytes);
779 sha1.update(bytes);
780 sha256.update(bytes);
781 sha512.update(bytes);
782 blake2::Digest::update(&mut blake2b, bytes);
783 if let Some(blocks) = &mut blocks {
784 blocks.update(bytes);
785 }
786 progress(done, size)
787 });
788 if let Some(error) = write_error {
789 return Err(error.into());
790 }
791 walked?;
792 let result = Verification {
793 bytes_verified: size,
794 md5: hex(&md5.finalize()),
795 sha1: hex(&sha1.finalize()),
796 sha256: hex(&sha256.finalize()),
797 sha512: hex(&sha512.finalize()),
798 blake2b: hex(&blake2::Digest::finalize(blake2b)),
799 references_match: None,
800 unsupported_hashes: Vec::new(),
801 };
802 let result = self.with_linear_references(id, result)?;
803 Ok((result, blocks.map(PairedBlockHasher::finish)))
804 }
805
806 fn with_linear_references(&self, id: &str, mut result: Verification) -> Result<Verification> {
807 result.references_match = None;
808 result.unsupported_hashes.clear();
809 for reference in self
810 .properties(id)
811 .filter(|p| is_property(&p.predicate, "hash"))
812 {
813 let datatype = reference.datatype.as_deref().unwrap_or("");
814 let expected = match datatype.strip_prefix(NS) {
815 Some("MD5") => &result.md5,
816 Some("SHA1") => &result.sha1,
817 Some("SHA256") => &result.sha256,
818 Some("SHA512") => &result.sha512,
819 Some("Blake2b" | "blake2b") => &result.blake2b,
820 _ => {
821 result.unsupported_hashes.push(datatype.into());
822 continue;
823 }
824 };
825 if reference.value.len() != expected.len()
826 || !reference.value.bytes().all(|b| b.is_ascii_hexdigit())
827 {
828 return Err(malformed("invalid linear hash"));
829 }
830 result.references_match = Some(
831 result.references_match.unwrap_or(true)
832 && reference.value.eq_ignore_ascii_case(expected),
833 );
834 }
835 Ok(result)
836 }
837
838 fn linear_backing(&mut self, id: &str) -> Result<String> {
843 let mut target = id.to_owned();
844 let mut visited = Vec::new();
845 loop {
846 enter(&target, &mut visited)?;
847 if self.inline_data(&target)?.is_some() {
848 return Ok(target);
849 }
850 if let Some(next) = self.value(&target, "dataStream")? {
851 target = next;
852 continue;
853 }
854 if self.has_type(&target, "Map") {
855 self.load_map(&target)?;
856 let map = &self.maps[&target];
857 if let [range] = map.ranges.as_slice()
858 && range.start == 0
859 && range.offset == 0
860 && range.end == self.size(&target)?
861 && self.size(&range.target).ok() == Some(range.end)
862 {
863 target = range.target.clone();
864 continue;
865 }
866 }
867 return Ok(target);
868 }
869 }
870
871 fn walk_bytes(
874 &mut self,
875 id: &str,
876 mut consume: impl FnMut(&[u8], u64) -> ControlFlow<()>,
877 ) -> Result<()> {
878 let size = self.size(id)?;
879 if consume(&[], 0).is_break() {
880 return Err(Error::Aborted);
881 }
882 self.read_inner(id, &mut [], 0, &mut Vec::new(), false)?;
883 let target = self.linear_backing(id)?;
884 let mut buffer = vec![0; 1024 * 1024];
885 if self.has_type(&target, "ZipSegment") || self.has_type(&target, "zip_segment") {
886 let path = self.path(&target)?;
887 let mut file = self.archive.by_name(&path)?;
888 if file.size() != size {
889 return Err(malformed("ZIP segment size mismatch"));
890 }
891 let mut offset = 0;
892 while offset < size {
893 let take = (size - offset).min(buffer.len() as u64) as usize;
894 file.read_exact(&mut buffer[..take])?;
895 offset += take as u64;
896 if consume(&buffer[..take], offset).is_break() {
897 return Err(Error::Aborted);
898 }
899 }
900 if file.read(&mut buffer[..1])? != 0 {
901 return Err(malformed("oversized ZIP segment"));
902 }
903 } else {
904 let mut offset = 0;
905 while offset < size {
906 let read = self.read_at_uncached(id, &mut buffer, offset)?;
907 if read == 0 {
908 return Err(malformed("truncated stream"));
909 }
910 offset += read as u64;
911 if consume(&buffer[..read], offset).is_break() {
912 return Err(Error::Aborted);
913 }
914 }
915 }
916 Ok(())
917 }
918
919 fn properties<'a>(&'a self, id: &str) -> impl Iterator<Item = &'a Property> {
920 self.graph.get(id).into_iter().flatten()
921 }
922 fn has_type(&self, id: &str, kind: &str) -> bool {
923 self.properties(id)
924 .any(|p| p.predicate == RDF_TYPE && is_property(&p.value, kind))
925 }
926 fn value(&self, id: &str, key: &str) -> Result<Option<String>> {
927 let values: BTreeSet<_> = self
928 .properties(id)
929 .filter(|p| is_property(&p.predicate, key))
930 .map(|p| p.value.clone())
931 .collect();
932 if values.len() > 1 {
933 return Err(malformed(format!("ambiguous {key}")));
934 }
935 Ok(values.into_iter().next())
936 }
937 fn number(&self, id: &str, key: &str) -> Result<u64> {
938 self.value(id, key)?
939 .ok_or_else(|| malformed(format!("missing {key}")))?
940 .parse()
941 .map_err(|_| malformed(format!("invalid {key}")))
942 }
943 fn stream_size(&self, id: &str, visited: &mut Vec<String>) -> Result<u64> {
944 enter(id, visited)?;
945 if self.inline_data(id)?.is_some() {
946 return self.number(id, "size");
947 }
948 if let Some(target) = self.value(id, "dataStream")? {
949 let size = self.stream_size(&target, visited)?;
950 if self.value(id, "size")?.is_some() && self.number(id, "size")? != size {
951 return Err(malformed("inconsistent stream size"));
952 }
953 Ok(size)
954 } else {
955 self.number(id, "size")
956 }
957 }
958 fn path(&self, id: &str) -> Result<String> {
959 if !self.has_type(id, "FileImage")
960 && let Some(path) = self.value(id, "fileName")?
961 {
962 return Ok(path);
963 }
964 storage_name(&self.archive, &self.volume, id, self.version)
965 }
966
967 fn inline_data(&self, id: &str) -> Result<Option<Vec<u8>>> {
968 let mut result = None;
969 for p in self.properties(id).filter(|p| {
970 (is_property(&p.predicate, "dataStream") || is_property(&p.predicate, "dataSteam"))
971 && p.datatype.as_deref() == Some(BASE64)
972 }) {
973 if p.value.len() > 1400 {
974 return Err(malformed("inline data exceeds 1 KiB limit"));
975 }
976 let bytes = base64::engine::general_purpose::STANDARD
977 .decode(&p.value)
978 .map_err(|_| malformed("invalid inline base64"))?;
979 if bytes.len() > 1024 || bytes.len() as u64 != self.number(id, "size")? {
980 return Err(malformed("inline data size mismatch"));
981 }
982 if result.as_ref().is_some_and(|old| old != &bytes) {
983 return Err(malformed("conflicting inline data"));
984 }
985 result = Some(bytes);
986 }
987 if result.is_some()
988 && self.properties(id).any(|p| {
989 (is_property(&p.predicate, "dataStream") || is_property(&p.predicate, "dataSteam"))
990 && p.datatype.as_deref() != Some(BASE64)
991 })
992 {
993 return Err(malformed("conflicting inline and referenced data streams"));
994 }
995 Ok(result)
996 }
997 fn load_map(&mut self, id: &str) -> Result<()> {
998 if self.maps.contains_key(id) {
999 return Ok(());
1000 }
1001 let path = self.path(id)?;
1002 let data = member(
1003 &mut self.archive,
1004 &format!("{path}/map"),
1005 self.limits.member_bytes,
1006 )?;
1007 let index = member(
1008 &mut self.archive,
1009 &format!("{path}/idx"),
1010 self.limits.metadata_bytes,
1011 )?;
1012 let index =
1013 std::str::from_utf8(&index).map_err(|_| malformed("map target index is not UTF-8"))?;
1014 let targets: Vec<_> = index.lines().collect();
1015 if data.len() % 28 != 0 {
1016 return Err(malformed("partial map record"));
1017 }
1018 let records = data.len() / 28;
1019 if records > self.limits.map_bytes / std::mem::size_of::<Range>() {
1020 return Err(malformed("retained map byte limit exceeded"));
1021 }
1022 let mut ranges = Vec::new();
1023 ranges
1024 .try_reserve_exact(records)
1025 .map_err(allocation_error)?;
1026 let mut retained = 0_usize;
1027 let mut previous = 0;
1028 let size = self.number(id, "size")?;
1029 for record in data.as_chunks::<28>().0 {
1030 let start = u64::from_le_bytes(record[..8].try_into().unwrap());
1031 let length = u64::from_le_bytes(record[8..16].try_into().unwrap());
1032 let offset = u64::from_le_bytes(record[16..24].try_into().unwrap());
1033 let target = u32::from_le_bytes(record[24..].try_into().unwrap()) as usize;
1034 let end = start
1035 .checked_add(length)
1036 .ok_or_else(|| malformed("map overflow"))?;
1037 if length == 0 || start < previous || end > size || offset.checked_add(length).is_none()
1038 {
1039 return Err(malformed("invalid or overlapping map range"));
1040 }
1041 let target = *targets
1042 .get(target)
1043 .ok_or_else(|| malformed("missing map target"))?;
1044 retained = retained
1045 .checked_add(std::mem::size_of::<Range>())
1046 .and_then(|n| n.checked_add(target.len()))
1047 .filter(|n| *n <= self.limits.map_bytes)
1048 .ok_or_else(|| malformed("retained map byte limit exceeded"))?;
1049 ranges.push(Range {
1050 start,
1051 end,
1052 offset,
1053 target: target.to_owned(),
1054 });
1055 previous = end;
1056 }
1057 self.maps.clear();
1059 self.maps.insert(
1060 id.into(),
1061 Arc::new(Map {
1062 ranges,
1063 gap: self
1064 .value(id, "mapGapDefaultStream")?
1065 .unwrap_or_else(|| format!("{NS}Zero")),
1066 }),
1067 );
1068 Ok(())
1069 }
1070
1071 fn read_stored_member_range(
1072 &mut self,
1073 name: &str,
1074 offset: u64,
1075 buffer: &mut [u8],
1076 ) -> Result<bool> {
1077 if self
1078 .stored_member
1079 .as_ref()
1080 .is_none_or(|member| member.name != name)
1081 {
1082 let directory_start = self.archive.central_directory_start();
1083 let file = self.archive.by_name(name)?;
1084 if file.size() > self.limits.member_bytes {
1085 return Err(malformed(format!("member exceeds resource limit: {name}")));
1086 }
1087 let size = file.size();
1088 let data_start = (file.compression() == CompressionMethod::Stored
1089 && !file.encrypted()
1090 && file.compressed_size() == size)
1091 .then(|| file.data_start())
1092 .flatten();
1093 if data_start
1094 .and_then(|start| start.checked_add(size))
1095 .is_some_and(|end| end > directory_start)
1096 {
1097 return Err(malformed("stored ZIP member exceeds archive data"));
1098 }
1099 self.stored_member = Some(StoredMember {
1100 name: name.to_owned(),
1101 size,
1102 data_start,
1103 });
1104 }
1105 let member = self.stored_member.as_ref().unwrap();
1106 if offset
1107 .checked_add(buffer.len() as u64)
1108 .is_none_or(|end| end > member.size)
1109 {
1110 return Err(malformed("member range exceeds stored ZIP entry"));
1111 }
1112 let Some(data_start) = member.data_start else {
1113 return Ok(false);
1114 };
1115 let absolute = data_start
1116 .checked_add(offset)
1117 .ok_or_else(|| malformed("stored ZIP member offset overflow"))?;
1118 read_file_exact_at(&self.stored_source, absolute, buffer)?;
1119 self.decoded_cache
1120 .lock()
1121 .map_err(|_| malformed("AFF4 decoded cache lock poisoned"))?
1122 .record_stored_member_range_read(buffer.len());
1123 Ok(true)
1124 }
1125
1126 fn read_inner(
1127 &mut self,
1128 id: &str,
1129 buffer: &mut [u8],
1130 offset: u64,
1131 visited: &mut Vec<String>,
1132 use_decoded_cache: bool,
1133 ) -> Result<()> {
1134 enter(id, visited)?;
1135 let result = self.read_inner_impl(id, buffer, offset, visited, use_decoded_cache);
1136 visited.pop();
1137 result
1138 }
1139 fn read_inner_impl(
1140 &mut self,
1141 id: &str,
1142 buffer: &mut [u8],
1143 offset: u64,
1144 visited: &mut Vec<String>,
1145 use_decoded_cache: bool,
1146 ) -> Result<()> {
1147 if id == format!("{NS}Zero") {
1148 buffer.fill(0);
1149 return Ok(());
1150 }
1151 if let Some(value) = id.strip_prefix(&format!("{NS}SymbolicStream")) {
1152 if value.len() != 2 {
1153 return Err(malformed("invalid symbolic byte"));
1154 }
1155 buffer.fill(
1156 u8::from_str_radix(value, 16).map_err(|_| malformed("invalid symbolic byte"))?,
1157 );
1158 return Ok(());
1159 }
1160 if id == format!("{NS}UnknownData") || id == format!("{NS}UnreadableData") {
1161 return Err(Error::Unsupported(
1162 "unknown or unreadable evidence range".into(),
1163 ));
1164 }
1165 let size = self.size(id)?;
1166 if offset
1167 .checked_add(buffer.len() as u64)
1168 .is_none_or(|end| end > size)
1169 {
1170 return Err(malformed("target range exceeds stream"));
1171 }
1172 if let Some(data) = self.inline_data(id)? {
1173 buffer.copy_from_slice(&data[offset as usize..offset as usize + buffer.len()]);
1174 return Ok(());
1175 }
1176 if let Some(target) = self.value(id, "dataStream")? {
1177 if self.has_type(id, "ContiguousImage") && self.has_type(&target, "Map") {
1178 self.load_map(&target)?;
1179 let map = &self.maps[&target];
1180 let mut end = 0;
1181 for range in &map.ranges {
1182 if range.start != end {
1183 return Err(malformed("gap in declared contiguous image"));
1184 }
1185 end = range.end;
1186 }
1187 if end != size {
1188 return Err(malformed("truncated contiguous image map"));
1189 }
1190 }
1191 return self.read_inner(&target, buffer, offset, visited, use_decoded_cache);
1192 }
1193 if self.has_type(id, "ZipSegment") || self.has_type(id, "zip_segment") {
1194 let path = self.path(id)?;
1195 let mut file = self.archive.by_name(&path)?;
1196 if file.size() != size {
1197 return Err(malformed("ZIP segment size mismatch"));
1198 }
1199 if std::io::copy(&mut file.by_ref().take(offset), &mut std::io::sink())? != offset {
1200 return Err(malformed("truncated ZIP segment"));
1201 }
1202 file.read_exact(buffer)?;
1203 if offset + buffer.len() as u64 == size {
1204 let mut eof = [0];
1205 if file.read(&mut eof)? != 0 {
1206 return Err(malformed("oversized ZIP segment"));
1207 }
1208 }
1209 return Ok(());
1210 }
1211 if self.has_type(id, "Map") {
1212 self.load_map(id)?;
1213 let map = self.maps[id].clone();
1214 let mut done = 0;
1215 while done < buffer.len() {
1216 let position = offset + done as u64;
1217 let next = map.ranges.partition_point(|r| r.end <= position);
1218 let (target, target_offset, end) = match map.ranges.get(next) {
1219 Some(range) if range.start <= position => (
1220 range.target.as_str(),
1221 range.offset + (position - range.start),
1222 range.end,
1223 ),
1224 next => (
1225 map.gap.as_str(),
1226 position,
1227 next.map_or(self.number(id, "size")?, |r| r.start),
1228 ),
1229 };
1230 let length = (end - position).min((buffer.len() - done) as u64) as usize;
1231 if length == 0 {
1232 return Err(malformed("map makes no progress"));
1233 }
1234 self.read_inner(
1235 target,
1236 &mut buffer[done..done + length],
1237 target_offset,
1238 visited,
1239 use_decoded_cache,
1240 )?;
1241 done += length;
1242 }
1243 return Ok(());
1244 }
1245 if !self.has_type(id, "ImageStream") {
1246 return Err(Error::Unsupported(format!("stream kind: {id}")));
1247 }
1248 let chunk_size = self.number(id, "chunkSize")?;
1249 let per_bevy = self.number(id, "chunksInSegment")?;
1250 if chunk_size == 0 || chunk_size > self.limits.chunk_bytes || per_bevy == 0 {
1251 return Err(malformed("invalid chunk geometry"));
1252 }
1253 let path = self.path(id)?;
1254 let compression = self.value(id, "compressionMethod")?;
1255 let mut done = 0;
1256 while done < buffer.len() {
1257 let position = offset + done as u64;
1258 let chunk = position / chunk_size;
1259 let cache_key = DecodedCacheKey::Chunk {
1260 volume: self.volume.clone(),
1261 resource: id.to_owned(),
1262 chunk,
1263 };
1264 let cached = if use_decoded_cache {
1265 self.decoded_cache
1266 .lock()
1267 .map_err(|_| malformed("AFF4 decoded cache lock poisoned"))?
1268 .get_chunk(&cache_key)
1269 } else {
1270 None
1271 };
1272 if let Some(decoded) = cached {
1273 let begin = (position % chunk_size) as usize;
1274 let take = (decoded.len() - begin).min(buffer.len() - done);
1275 buffer[done..done + take].copy_from_slice(&decoded[begin..begin + take]);
1276 done += take;
1277 continue;
1278 }
1279 let name = format!("{path}/{:08}", chunk / per_bevy);
1280 if self
1281 .index_cache
1282 .as_ref()
1283 .is_none_or(|(key, _)| key != &name)
1284 {
1285 self.index_cache = Some((
1286 name.clone(),
1287 member(
1288 &mut self.archive,
1289 &format!("{name}.index"),
1290 self.limits.member_bytes,
1291 )?,
1292 ));
1293 }
1294 let index = &self.index_cache.as_ref().unwrap().1;
1295 if !index.len().is_multiple_of(12) {
1296 return Err(malformed("partial chunk index record"));
1297 }
1298 let index_offset = usize::try_from(chunk % per_bevy)
1299 .ok()
1300 .and_then(|n| n.checked_mul(12))
1301 .ok_or_else(|| malformed("index overflow"))?;
1302 let index_end = index_offset
1303 .checked_add(12)
1304 .ok_or_else(|| malformed("index overflow"))?;
1305 let record = index
1306 .get(index_offset..index_end)
1307 .ok_or_else(|| malformed("missing chunk index"))?;
1308 let start = u64::from_le_bytes(record[..8].try_into().unwrap());
1309 let length = u32::from_le_bytes(record[8..].try_into().unwrap()) as u64;
1310 let end = start
1311 .checked_add(length)
1312 .filter(|end| *end <= self.limits.member_bytes)
1313 .ok_or_else(|| malformed("chunk outside bevy"))?;
1314 let mut direct = chunk_buffer(length as usize)?;
1315 let direct_read = self.read_stored_member_range(&name, start, &mut direct)?;
1316 if direct_read {
1317 self.cache = None;
1318 } else if self.cache.as_ref().is_none_or(|(key, _)| key != &name) {
1319 self.cache = Some((
1320 name.clone(),
1321 member(&mut self.archive, &name, self.limits.member_bytes)?,
1322 ));
1323 }
1324 let encoded = if direct_read {
1325 direct.as_slice()
1326 } else {
1327 let data = &self.cache.as_ref().unwrap().1;
1328 if end > data.len() as u64 {
1329 return Err(malformed("chunk outside bevy"));
1330 }
1331 &data[start as usize..end as usize]
1332 };
1333 let decode_started = Instant::now();
1334 let decoded = if length == chunk_size {
1335 if direct_read {
1336 direct
1337 } else {
1338 encoded.to_vec()
1339 }
1340 } else {
1341 decode(encoded, compression.as_deref(), chunk_size as usize)?
1342 };
1343 self.decoded_cache
1344 .lock()
1345 .map_err(|_| malformed("AFF4 decoded cache lock poisoned"))?
1346 .record_decode(decoded.len(), decode_started.elapsed());
1347 if decoded.len() != chunk_size as usize {
1348 return Err(malformed("decoded chunk length mismatch"));
1349 }
1350 let begin = (position % chunk_size) as usize;
1351 let take = (decoded.len() - begin).min(buffer.len() - done);
1352 buffer[done..done + take].copy_from_slice(&decoded[begin..begin + take]);
1353 done += take;
1354 if use_decoded_cache {
1355 self.decoded_cache
1356 .lock()
1357 .map_err(|_| malformed("AFF4 decoded cache lock poisoned"))?
1358 .insert(cache_key, Arc::new(decoded));
1359 }
1360 }
1361 Ok(())
1362 }
1363}
1364
1365#[derive(Clone)]
1366pub(super) struct PairedBlockDigests {
1367 md5: Vec<[u8; 16]>,
1368 sha256: Vec<[u8; 32]>,
1369}
1370
1371struct PairedBlockHasher {
1372 block_size: usize,
1373 used: usize,
1374 md5: Md5,
1375 sha256: Sha256,
1376 result: PairedBlockDigests,
1377}
1378
1379impl PairedBlockHasher {
1380 fn new(block_size: usize) -> Self {
1381 debug_assert!(block_size > 0);
1382 Self {
1383 block_size,
1384 used: 0,
1385 md5: Md5::new(),
1386 sha256: Sha256::new(),
1387 result: PairedBlockDigests {
1388 md5: Vec::new(),
1389 sha256: Vec::new(),
1390 },
1391 }
1392 }
1393
1394 fn update(&mut self, mut bytes: &[u8]) {
1395 while !bytes.is_empty() {
1396 let take = bytes.len().min(self.block_size - self.used);
1397 self.md5.update(&bytes[..take]);
1398 self.sha256.update(&bytes[..take]);
1399 self.used += take;
1400 bytes = &bytes[take..];
1401 if self.used == self.block_size {
1402 self.finish_block();
1403 }
1404 }
1405 }
1406
1407 fn finish(mut self) -> PairedBlockDigests {
1408 if self.used != 0 {
1409 self.finish_block();
1410 }
1411 self.result
1412 }
1413
1414 fn finish_block(&mut self) {
1415 self.result.md5.push(
1416 std::mem::replace(&mut self.md5, Md5::new())
1417 .finalize()
1418 .into(),
1419 );
1420 self.result.sha256.push(
1421 std::mem::replace(&mut self.sha256, Sha256::new())
1422 .finalize()
1423 .into(),
1424 );
1425 self.used = 0;
1426 }
1427}
1428
1429#[cfg(unix)]
1430fn read_file_exact_at(file: &File, offset: u64, buffer: &mut [u8]) -> std::io::Result<()> {
1431 use std::os::unix::fs::FileExt;
1432 file.read_exact_at(buffer, offset)
1433}
1434
1435#[cfg(windows)]
1436fn read_file_exact_at(file: &File, offset: u64, mut buffer: &mut [u8]) -> std::io::Result<()> {
1437 use std::os::windows::fs::FileExt;
1438 let mut position = offset;
1439 while !buffer.is_empty() {
1440 let read = file.seek_read(buffer, position)?;
1441 if read == 0 {
1442 return Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof));
1443 }
1444 position = position
1445 .checked_add(read as u64)
1446 .ok_or_else(|| std::io::Error::from(std::io::ErrorKind::InvalidInput))?;
1447 buffer = &mut buffer[read..];
1448 }
1449 Ok(())
1450}
1451
1452#[cfg(not(any(unix, windows)))]
1453fn read_file_exact_at(file: &File, offset: u64, buffer: &mut [u8]) -> std::io::Result<()> {
1454 use std::io::{Seek, SeekFrom};
1455 let mut file = file.try_clone()?;
1456 file.seek(SeekFrom::Start(offset))?;
1457 file.read_exact(buffer)
1458}
1459
1460fn enter(id: &str, visited: &mut Vec<String>) -> Result<()> {
1461 if visited.len() >= 32 || visited.iter().any(|v| v == id) {
1462 return Err(malformed("cyclic or excessively deep stream references"));
1463 }
1464 visited.push(id.to_owned());
1465 Ok(())
1466}
1467
1468fn is_property(predicate: &str, local: &str) -> bool {
1469 [NS, LOGICAL_NS, LEGACY_LOGICAL_NS]
1470 .iter()
1471 .any(|ns| predicate.strip_prefix(ns) == Some(local))
1472}
1473
1474fn storage_name(
1475 archive: &ZipArchive<File>,
1476 volume: &str,
1477 id: &str,
1478 version: (u32, u32),
1479) -> Result<String> {
1480 if version != (1, 0) && archive.index_for_name(id).is_some() {
1481 return Ok(id.to_owned());
1482 }
1483 if let Some(path) = id.strip_prefix(&format!("{volume}/")) {
1484 return Ok(if version == (1, 0) {
1485 escape(path)
1486 } else {
1487 path.to_owned()
1488 });
1489 }
1490 let rest = id
1491 .strip_prefix("aff4://")
1492 .ok_or_else(|| Error::Unsupported("non-AFF4 storage reference".into()))?;
1493 let legacy = format!("aff4%3A%2F%2F{}", escape(rest));
1496 if version == (1, 1)
1497 && archive
1498 .file_names()
1499 .any(|n| n.starts_with(&format!("{legacy}/")))
1500 {
1501 return Ok(legacy);
1502 }
1503 Ok(if version == (1, 0) {
1504 format!("aff4%3A%2F%2F{}", escape(rest))
1505 } else {
1506 id.to_owned()
1507 })
1508}
1509fn member(archive: &mut ZipArchive<File>, name: &str, limit: u64) -> Result<Vec<u8>> {
1510 let file = archive.by_name(name)?;
1511 if file.size() > limit {
1512 return Err(malformed(format!("member exceeds resource limit: {name}")));
1513 }
1514 read_buffer(file, limit)
1516}
1517
1518fn allocation_error(error: std::collections::TryReserveError) -> Error {
1519 std::io::Error::new(std::io::ErrorKind::OutOfMemory, error).into()
1520}
1521
1522fn read_buffer(mut reader: impl Read, limit: u64) -> Result<Vec<u8>> {
1523 let mut data = Vec::new();
1524 let mut buffer = [0; 64 * 1024];
1525 loop {
1526 let take = (limit.saturating_sub(data.len() as u64).saturating_add(1))
1527 .min(buffer.len() as u64) as usize;
1528 let count = match reader.read(&mut buffer[..take]) {
1529 Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,
1530 value => value?,
1531 };
1532 if count == 0 {
1533 return Ok(data);
1534 }
1535 if count as u64 > limit.saturating_sub(data.len() as u64) {
1536 return Err(malformed("decoded member exceeds limit"));
1537 }
1538 data.try_reserve(count).map_err(allocation_error)?;
1539 data.extend_from_slice(&buffer[..count]);
1540 }
1541}
1542fn decode(data: &[u8], method: Option<&str>, size: usize) -> Result<Vec<u8>> {
1543 match method {
1544 Some("http://code.google.com/p/snappy/") => {
1545 if snap::raw::decompress_len(data).map_err(|e| malformed(e.to_string()))? != size {
1546 return Err(malformed("snappy size mismatch"));
1547 }
1548 let mut decoded = chunk_buffer(size)?;
1549 let count = snap::raw::Decoder::new()
1550 .decompress(data, &mut decoded)
1551 .map_err(|e| malformed(e.to_string()))?;
1552 decoded.truncate(count);
1553 Ok(decoded)
1554 }
1555 Some("https://www.ietf.org/rfc/rfc1950.txt") => {
1556 read_buffer(flate2::read::ZlibDecoder::new(data), size as u64)
1557 }
1558 Some("https://tools.ietf.org/html/rfc1951") => {
1559 read_buffer(flate2::read::DeflateDecoder::new(data), size as u64)
1560 }
1561 Some("https://code.google.com/p/lz4/") => {
1562 let mut decoded = chunk_buffer(size)?;
1563 let count = lz4_flex::block::decompress_into(data, &mut decoded)
1564 .map_err(|e| malformed(e.to_string()))?;
1565 decoded.truncate(count);
1566 Ok(decoded)
1567 }
1568 _ => Err(Error::Unsupported("chunk compression method".into())),
1569 }
1570}
1571
1572fn chunk_buffer(size: usize) -> Result<Vec<u8>> {
1573 let mut buffer = Vec::new();
1574 buffer.try_reserve_exact(size).map_err(allocation_error)?;
1575 buffer.resize(size, 0);
1576 Ok(buffer)
1577}
1578pub(crate) fn hex(data: &[u8]) -> String {
1579 data.iter().map(|b| format!("{b:02x}")).collect()
1580}
1581fn escape(value: &str) -> String {
1582 let mut output = String::new();
1583 for b in value.bytes() {
1584 if b.is_ascii_alphanumeric() || b"-._~/".contains(&b) {
1585 output.push(char::from(b));
1586 } else {
1587 output.push_str(&format!("%{b:02X}"));
1588 }
1589 }
1590 output
1591}
1592
1593#[cfg(test)]
1594mod tests {
1595 use super::*;
1596
1597 fn chunk(chunk: u64) -> DecodedCacheKey {
1598 DecodedCacheKey::Chunk {
1599 volume: "aff4://volume".into(),
1600 resource: "aff4://volume/data".into(),
1601 chunk,
1602 }
1603 }
1604
1605 #[test]
1606 fn decoded_cache_is_byte_bounded_and_lru() {
1607 let mut cache = DecodedChunkCache::with_capacity(8);
1608 cache.insert(chunk(0), Arc::new(vec![0; 4]));
1609 cache.insert(chunk(1), Arc::new(vec![1; 4]));
1610 assert!(cache.get_chunk(&chunk(0)).is_some());
1611 cache.insert(chunk(2), Arc::new(vec![2; 4]));
1612
1613 assert!(cache.get_chunk(&chunk(0)).is_some());
1614 assert!(cache.get_chunk(&chunk(1)).is_none());
1615 assert!(cache.get_chunk(&chunk(2)).is_some());
1616 assert_eq!(cache.current_bytes, 8);
1617 assert_eq!(cache.statistics.decoded_cache_evictions, 1);
1618 assert_eq!(cache.info().capacity_bytes(), 8);
1619 }
1620
1621 #[test]
1622 fn oversized_entry_is_not_retained() {
1623 let mut cache = DecodedChunkCache::with_capacity(4);
1624 cache.insert(chunk(0), Arc::new(vec![0; 5]));
1625 assert_eq!(cache.current_bytes, 0);
1626 assert_eq!(cache.entries.len(), 0);
1627 }
1628}