Skip to main content

aff4_image/
reader.rs

1use std::collections::{BTreeMap, BTreeSet};
2use std::fs::File;
3use std::io::{Read, Write};
4use std::ops::ControlFlow;
5use std::path::Path;
6use std::sync::{Arc, Mutex};
7use std::time::{Duration, Instant};
8
9use lru::LruCache;
10use md5::{Digest, Md5};
11use oxrdf::{NamedOrBlankNode, Term};
12use oxttl::TurtleParser;
13use sha1::Sha1;
14use sha2::Sha256;
15use zip::{CompressionMethod, ZipArchive};
16
17use crate::{Error, Result, malformed};
18use base64::Engine;
19
20#[path = "integrity.rs"]
21mod integrity;
22pub use integrity::{CheckOutcome, IntegrityCheck, MetadataVerification};
23
24#[path = "verify_all.rs"]
25mod verify_all;
26pub use verify_all::{ByteCoverage, ContainerVerification, ResourceVerification};
27
28#[path = "scan.rs"]
29mod scan;
30pub use scan::MetadataScan;
31
32#[path = "archive.rs"]
33mod archive;
34
35#[path = "volume_set.rs"]
36mod volume_set;
37pub use volume_set::{SetDigest, SetVerification, SetVolumeVerification, VolumeSet, VolumeSource};
38
39#[path = "discovery.rs"]
40mod discovery;
41pub use discovery::{BackingVolume, DiskImageSet, PhysicalDisk, PhysicalDiskReader};
42
43const NS: &str = "http://aff4.org/Schema#";
44const RDF_TYPE: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type";
45const LOGICAL_NS: &str = "https://aff4.org/Schema/2022/#";
46const LEGACY_LOGICAL_NS: &str = "http://aff4.org/Schema/2022/#";
47const BASE64: &str = "http://www.w3.org/2001/XMLSchema#base64Binary";
48const DECODED_CACHE_CAPACITY_BYTES: usize = 128 * 1024 * 1024;
49const LOGICAL_READ_AHEAD_BYTES: usize = 1024 * 1024;
50
51#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize)]
52#[non_exhaustive]
53/// Cumulative performance counters for one shared AFF4 payload cache.
54///
55/// Physical readers for disks in the same discovered volume set share these
56/// counters and the cache itself.
57pub struct ReaderStatistics {
58    decoded_cache_hits: u64,
59    decoded_cache_misses: u64,
60    read_ahead_cache_hits: u64,
61    read_ahead_cache_misses: u64,
62    read_ahead_prefetches: u64,
63    decoded_cache_evictions: u64,
64    decoded_bytes: u64,
65    decode_nanos: u64,
66    stored_member_range_reads: u64,
67    stored_member_range_bytes: u64,
68}
69
70impl ReaderStatistics {
71    /// Returns decoded chunk-cache hits.
72    pub fn decoded_cache_hits(&self) -> u64 {
73        self.decoded_cache_hits
74    }
75
76    /// Returns decoded chunk-cache misses.
77    pub fn decoded_cache_misses(&self) -> u64 {
78        self.decoded_cache_misses
79    }
80
81    /// Returns logical read-ahead page-cache hits.
82    pub fn read_ahead_cache_hits(&self) -> u64 {
83        self.read_ahead_cache_hits
84    }
85
86    /// Returns logical read-ahead page-cache misses.
87    pub fn read_ahead_cache_misses(&self) -> u64 {
88        self.read_ahead_cache_misses
89    }
90
91    /// Returns logical pages fetched automatically after small positioned reads.
92    pub fn read_ahead_prefetches(&self) -> u64 {
93        self.read_ahead_prefetches
94    }
95
96    /// Returns decoded pages or chunks evicted to retain the automatic memory bound.
97    pub fn decoded_cache_evictions(&self) -> u64 {
98        self.decoded_cache_evictions
99    }
100
101    /// Returns logical chunk bytes decoded after cache misses.
102    pub fn decoded_bytes(&self) -> u64 {
103        self.decoded_bytes
104    }
105
106    /// Returns nanoseconds spent decoding chunks after cache misses.
107    pub fn decode_nanos(&self) -> u64 {
108        self.decode_nanos
109    }
110
111    /// Returns direct range reads from uncompressed ZIP members.
112    pub fn stored_member_range_reads(&self) -> u64 {
113        self.stored_member_range_reads
114    }
115
116    /// Returns bytes read directly from uncompressed ZIP members.
117    pub fn stored_member_range_bytes(&self) -> u64 {
118        self.stored_member_range_bytes
119    }
120
121    /// Returns a field-wise saturating delta from an earlier snapshot.
122    #[must_use]
123    pub fn saturating_delta(self, earlier: Self) -> Self {
124        Self {
125            decoded_cache_hits: self
126                .decoded_cache_hits
127                .saturating_sub(earlier.decoded_cache_hits),
128            decoded_cache_misses: self
129                .decoded_cache_misses
130                .saturating_sub(earlier.decoded_cache_misses),
131            read_ahead_cache_hits: self
132                .read_ahead_cache_hits
133                .saturating_sub(earlier.read_ahead_cache_hits),
134            read_ahead_cache_misses: self
135                .read_ahead_cache_misses
136                .saturating_sub(earlier.read_ahead_cache_misses),
137            read_ahead_prefetches: self
138                .read_ahead_prefetches
139                .saturating_sub(earlier.read_ahead_prefetches),
140            decoded_cache_evictions: self
141                .decoded_cache_evictions
142                .saturating_sub(earlier.decoded_cache_evictions),
143            decoded_bytes: self.decoded_bytes.saturating_sub(earlier.decoded_bytes),
144            decode_nanos: self.decode_nanos.saturating_sub(earlier.decode_nanos),
145            stored_member_range_reads: self
146                .stored_member_range_reads
147                .saturating_sub(earlier.stored_member_range_reads),
148            stored_member_range_bytes: self
149                .stored_member_range_bytes
150                .saturating_sub(earlier.stored_member_range_bytes),
151        }
152    }
153}
154
155#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize)]
156#[non_exhaustive]
157/// Configured and observed payload bytes for the automatic AFF4 reader cache.
158pub struct ReaderCacheInfo {
159    capacity_bytes: u64,
160    current_bytes: u64,
161    peak_bytes: u64,
162    entries: u64,
163}
164
165impl ReaderCacheInfo {
166    /// Returns the automatic decoded payload-cache byte ceiling.
167    pub fn capacity_bytes(&self) -> u64 {
168        self.capacity_bytes
169    }
170
171    /// Returns currently retained decoded page and chunk bytes.
172    pub fn current_bytes(&self) -> u64 {
173        self.current_bytes
174    }
175
176    /// Returns peak retained decoded page and chunk bytes.
177    pub fn peak_bytes(&self) -> u64 {
178        self.peak_bytes
179    }
180
181    /// Returns the number of retained decoded pages and chunks.
182    pub fn entries(&self) -> u64 {
183        self.entries
184    }
185}
186
187#[derive(Clone, Hash, PartialEq, Eq)]
188enum DecodedCacheKey {
189    Chunk {
190        volume: String,
191        resource: String,
192        chunk: u64,
193    },
194    DiskPage {
195        volume: String,
196        resource: String,
197        offset: u64,
198    },
199}
200
201struct DecodedChunkCache {
202    entries: LruCache<DecodedCacheKey, Arc<Vec<u8>>>,
203    capacity_bytes: usize,
204    current_bytes: usize,
205    peak_bytes: usize,
206    statistics: ReaderStatistics,
207}
208
209impl DecodedChunkCache {
210    fn new() -> Self {
211        Self::with_capacity(DECODED_CACHE_CAPACITY_BYTES)
212    }
213
214    fn with_capacity(capacity_bytes: usize) -> Self {
215        Self {
216            entries: LruCache::unbounded(),
217            capacity_bytes,
218            current_bytes: 0,
219            peak_bytes: 0,
220            statistics: ReaderStatistics::default(),
221        }
222    }
223
224    fn get_chunk(&mut self, key: &DecodedCacheKey) -> Option<Arc<Vec<u8>>> {
225        let result = self.entries.get(key).cloned();
226        if result.is_some() {
227            self.statistics.decoded_cache_hits =
228                self.statistics.decoded_cache_hits.saturating_add(1);
229        } else {
230            self.statistics.decoded_cache_misses =
231                self.statistics.decoded_cache_misses.saturating_add(1);
232        }
233        result
234    }
235
236    fn get_page(&mut self, key: &DecodedCacheKey) -> Option<Arc<Vec<u8>>> {
237        let result = self.entries.get(key).cloned();
238        if result.is_some() {
239            self.statistics.read_ahead_cache_hits =
240                self.statistics.read_ahead_cache_hits.saturating_add(1);
241        } else {
242            self.statistics.read_ahead_cache_misses =
243                self.statistics.read_ahead_cache_misses.saturating_add(1);
244        }
245        result
246    }
247
248    fn insert(&mut self, key: DecodedCacheKey, decoded: Arc<Vec<u8>>) {
249        let bytes = decoded.len();
250        if bytes > self.capacity_bytes {
251            return;
252        }
253        if let Some(previous) = self.entries.put(key, decoded) {
254            self.current_bytes = self.current_bytes.saturating_sub(previous.len());
255        }
256        self.current_bytes = self.current_bytes.saturating_add(bytes);
257        while self.current_bytes > self.capacity_bytes {
258            let Some((_, evicted)) = self.entries.pop_lru() else {
259                break;
260            };
261            self.current_bytes = self.current_bytes.saturating_sub(evicted.len());
262            self.statistics.decoded_cache_evictions =
263                self.statistics.decoded_cache_evictions.saturating_add(1);
264        }
265        self.peak_bytes = self.peak_bytes.max(self.current_bytes);
266    }
267
268    fn record_decode(&mut self, bytes: usize, elapsed: Duration) {
269        self.statistics.decoded_bytes = self
270            .statistics
271            .decoded_bytes
272            .saturating_add(u64::try_from(bytes).unwrap_or(u64::MAX));
273        self.statistics.decode_nanos = self
274            .statistics
275            .decode_nanos
276            .saturating_add(u64::try_from(elapsed.as_nanos()).unwrap_or(u64::MAX));
277    }
278
279    fn record_read_ahead_prefetch(&mut self) {
280        self.statistics.read_ahead_prefetches =
281            self.statistics.read_ahead_prefetches.saturating_add(1);
282    }
283
284    fn record_stored_member_range_read(&mut self, bytes: usize) {
285        self.statistics.stored_member_range_reads =
286            self.statistics.stored_member_range_reads.saturating_add(1);
287        self.statistics.stored_member_range_bytes = self
288            .statistics
289            .stored_member_range_bytes
290            .saturating_add(u64::try_from(bytes).unwrap_or(u64::MAX));
291    }
292
293    fn clear(&mut self) {
294        self.entries.clear();
295        self.current_bytes = 0;
296    }
297
298    fn info(&self) -> ReaderCacheInfo {
299        ReaderCacheInfo {
300            capacity_bytes: self.capacity_bytes as u64,
301            current_bytes: self.current_bytes as u64,
302            peak_bytes: self.peak_bytes as u64,
303            entries: self.entries.len() as u64,
304        }
305    }
306}
307
308/// Limits on retained metadata, map/index members, and decoded chunks.
309#[derive(Debug, Clone, serde::Serialize)]
310pub struct Limits {
311    /// Maximum ZIP central-directory bytes, checked before ZIP parsing.
312    pub directory_bytes: u64,
313    /// Maximum ZIP entries, checked before ZIP parsing.
314    pub archive_entries: usize,
315    /// Maximum decompressed metadata bytes.
316    pub metadata_bytes: u64,
317    /// Maximum bytes in an individual retained data/index member.
318    pub member_bytes: u64,
319    /// Maximum decoded chunk size.
320    pub chunk_bytes: u64,
321    /// Maximum RDF triples retained.
322    pub triples: usize,
323    /// Maximum retained map range records and target strings, in bytes.
324    pub map_bytes: usize,
325    /// Maximum logical bytes traversed by one verification/copy operation.
326    /// Container-wide verification shares this across resources and block scans.
327    pub verification_bytes: u64,
328}
329impl Default for Limits {
330    fn default() -> Self {
331        Self {
332            directory_bytes: 64 * 1024 * 1024,
333            archive_entries: 200_000,
334            metadata_bytes: 16 * 1024 * 1024,
335            member_bytes: 128 * 1024 * 1024,
336            chunk_bytes: 16 * 1024 * 1024,
337            triples: 200_000,
338            map_bytes: 128 * 1024 * 1024,
339            verification_bytes: 64 * 1024 * 1024 * 1024 * 1024,
340        }
341    }
342}
343
344impl Limits {
345    /// Uses platform capacity instead of application resource quotas.
346    ///
347    /// Intended for local CLI operations. This does not reserve memory or promise
348    /// recovery from process-wide memory exhaustion. Format validation, checked
349    /// arithmetic, and dependency-cycle checks still apply. Library callers
350    /// processing untrusted inputs should normally retain explicit limits.
351    pub const fn unrestricted() -> Self {
352        Self {
353            directory_bytes: isize::MAX as u64,
354            archive_entries: isize::MAX as usize,
355            metadata_bytes: isize::MAX as u64,
356            member_bytes: isize::MAX as u64,
357            chunk_bytes: isize::MAX as u64,
358            triples: isize::MAX as usize,
359            map_bytes: isize::MAX as usize,
360            verification_bytes: u64::MAX,
361        }
362    }
363}
364
365/// Preserved RDF property; values remain associated with their subject.
366#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
367pub struct Property {
368    /// Fully expanded predicate IRI.
369    pub predicate: String,
370    /// IRI, blank-node label, or literal lexical value.
371    pub value: String,
372    /// Literal datatype IRI; absent for resource references.
373    pub datatype: Option<String>,
374    /// Literal language tag, when present.
375    pub language: Option<String>,
376}
377
378/// An explicitly selectable image or underlying data stream.
379#[derive(Debug, Clone, serde::Serialize)]
380pub struct StreamInfo {
381    /// AFF4 resource identifier.
382    pub id: String,
383    /// RDF classes on the resource.
384    pub types: Vec<String>,
385    /// Logical byte size when supplied or resolvable from its data stream.
386    pub size: Option<u64>,
387}
388
389/// Selected-stream linear digest results. Unsupported references are explicit.
390#[derive(Debug, Clone, serde::Serialize)]
391pub struct Verification {
392    /// Number of logical bytes read.
393    pub bytes_verified: u64,
394    /// Computed hexadecimal MD5.
395    pub md5: String,
396    /// Computed hexadecimal SHA1.
397    pub sha1: String,
398    /// Computed hexadecimal SHA256.
399    pub sha256: String,
400    /// Computed hexadecimal SHA512.
401    pub sha512: String,
402    /// Computed hexadecimal BLAKE2b-512.
403    pub blake2b: String,
404    /// Available supported linear references all match; None means none exist.
405    pub references_match: Option<bool>,
406    /// Datatypes of stored hashes not verified, including block-map hashes.
407    pub unsupported_hashes: Vec<String>,
408}
409
410/// One read-only ZIP container. Methods serialize access through `&mut self`.
411/// Open does not certify contents. Backing files must remain unchanged.
412pub struct Container {
413    archive: ZipArchive<File>,
414    stored_source: File,
415    volume: String,
416    graph: BTreeMap<String, Vec<Property>>,
417    limits: Limits,
418    cache: Option<(String, Vec<u8>)>,
419    index_cache: Option<(String, Vec<u8>)>,
420    stored_member: Option<StoredMember>,
421    decoded_cache: Arc<Mutex<DecodedChunkCache>>,
422    maps: BTreeMap<String, Arc<Map>>,
423    version: (u32, u32),
424    metadata_members: BTreeMap<String, String>,
425    usage: archive::Usage,
426}
427
428struct StoredMember {
429    name: String,
430    size: u64,
431    data_start: Option<u64>,
432}
433
434struct PositionedReader<'a> {
435    container: &'a mut Container,
436    id: String,
437    position: u64,
438}
439
440impl Read for PositionedReader<'_> {
441    fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
442        let count = self
443            .container
444            .read_at(&self.id, buffer, self.position)
445            .map_err(std::io::Error::other)?;
446        self.position += count as u64;
447        Ok(count)
448    }
449}
450
451#[derive(Clone)]
452struct Range {
453    start: u64,
454    end: u64,
455    offset: u64,
456    target: String,
457}
458#[derive(Clone)]
459struct Map {
460    ranges: Vec<Range>,
461    gap: String,
462}
463
464impl Container {
465    /// Opens a standard v1 ZIP container with default resource limits.
466    pub fn open(path: impl AsRef<Path>) -> Result<Self> {
467        Self::open_with_limits(path, Limits::default())
468    }
469
470    /// Opens a container, validates its version, and parses bounded RDF metadata.
471    /// External RDF references are never fetched over the network.
472    pub fn open_with_limits(path: impl AsRef<Path>, limits: Limits) -> Result<Self> {
473        let (mut archive, mut usage) = archive::open(path.as_ref(), &limits)?;
474        let stored_source = File::open(path.as_ref())?;
475        let mut names = BTreeSet::new();
476        for name in archive.file_names() {
477            if !names.insert(name.to_owned()) {
478                return Err(malformed("duplicate ZIP member name"));
479            }
480        }
481        let version = member(&mut archive, "version.txt", 4096)?;
482        let version =
483            std::str::from_utf8(&version).map_err(|_| malformed("invalid version text"))?;
484        let mut fields = BTreeMap::new();
485        for line in version.lines() {
486            if let Some((key, value)) = line.split_once('=')
487                && fields.insert(key.trim(), value.trim()).is_some()
488            {
489                return Err(malformed("duplicate version field"));
490            }
491        }
492        let version = (
493            fields
494                .get("major")
495                .and_then(|n| n.parse().ok())
496                .ok_or_else(|| malformed("missing or invalid major version"))?,
497            fields
498                .get("minor")
499                .and_then(|n| n.parse().ok())
500                .ok_or_else(|| malformed("missing or invalid minor version"))?,
501        );
502        if !matches!(version, (1, 0) | (1, 1) | (2, 1)) {
503            return Err(Error::Unsupported(
504                "container version (supported: 1.0, legacy logical 1.1, draft logical 2.1)".into(),
505            ));
506        }
507        let comment = std::str::from_utf8(archive.comment())
508            .map_err(|_| malformed("volume comment is not UTF-8"))?
509            .trim_end_matches('\0')
510            .trim()
511            .to_owned();
512        let description = match archive.index_for_name("container.description") {
513            Some(_) => String::from_utf8(member(&mut archive, "container.description", 4096)?)
514                .map_err(|_| malformed("volume name is not UTF-8"))?,
515            None => comment.clone(),
516        };
517        let volume = description.trim_end_matches('\0').trim().to_owned();
518        if !volume.starts_with("aff4://") || volume.len() <= 7 {
519            return Err(malformed("missing AFF4 volume URI"));
520        }
521        if !comment.is_empty() && comment != volume {
522            return Err(malformed("conflicting volume identifiers"));
523        }
524        let mut graph: BTreeMap<String, Vec<Property>> = BTreeMap::new();
525        let mut pending = vec!["information.turtle".to_owned()];
526        let mut seen = BTreeSet::new();
527        let mut metadata_members = BTreeMap::new();
528        let mut remaining = limits.metadata_bytes;
529        let mut count = 0;
530        while let Some(name) = pending.pop() {
531            if !seen.insert(name.clone()) {
532                continue;
533            }
534            let turtle = member(&mut archive, &name, remaining)?;
535            remaining -= turtle.len() as u64;
536            metadata_members.insert(name.clone(), hex(&Sha256::digest(&turtle)));
537            for triple in TurtleParser::new().for_reader(turtle.as_slice()) {
538                if count >= limits.triples {
539                    return Err(malformed("RDF triple limit exceeded"));
540                }
541                let triple = triple.map_err(|error| malformed(format!("Turtle: {error}")))?;
542                count += 1;
543                let subject = match triple.subject {
544                    NamedOrBlankNode::NamedNode(n) => n.into_string(),
545                    NamedOrBlankNode::BlankNode(n) => format!("_:{n}"),
546                };
547                let (value, datatype, language) = match triple.object {
548                    Term::NamedNode(n) => (n.into_string(), None, None),
549                    Term::BlankNode(n) => (format!("_:{n}"), None, None),
550                    Term::Literal(l) => (
551                        l.value().to_owned(),
552                        Some(l.datatype().as_str().to_owned()),
553                        l.language().map(str::to_owned),
554                    ),
555                };
556                if is_property(triple.predicate.as_str(), "imports") {
557                    if datatype.is_some() {
558                        return Err(malformed("metadata import must be an IRI"));
559                    }
560                    pending.push(storage_name(&archive, &volume, &value, version)?);
561                }
562                graph.entry(subject).or_default().push(Property {
563                    predicate: triple.predicate.into_string(),
564                    value,
565                    datatype,
566                    language,
567                });
568            }
569        }
570        usage.metadata_bytes = limits.metadata_bytes - remaining;
571        usage.triples = count;
572        Ok(Self {
573            archive,
574            stored_source,
575            volume,
576            graph,
577            limits,
578            cache: None,
579            index_cache: None,
580            stored_member: None,
581            decoded_cache: Arc::new(Mutex::new(DecodedChunkCache::new())),
582            maps: BTreeMap::new(),
583            version,
584            metadata_members,
585            usage,
586        })
587    }
588
589    /// Preserves all parsed RDF properties, including unsupported extensions.
590    pub fn metadata(&self) -> &BTreeMap<String, Vec<Property>> {
591        &self.graph
592    }
593
594    /// Returns the declared volume identifier.
595    pub fn volume_id(&self) -> &str {
596        &self.volume
597    }
598
599    /// Declared container version; 2.1 refers to the evolving AFF4-L draft.
600    pub fn version(&self) -> (u32, u32) {
601        self.version
602    }
603
604    /// Returns cumulative decoded-chunk reader counters.
605    pub fn reader_statistics(&self) -> ReaderStatistics {
606        self.decoded_cache
607            .lock()
608            .unwrap_or_else(std::sync::PoisonError::into_inner)
609            .statistics
610    }
611
612    /// Returns automatic decoded-chunk cache usage.
613    pub fn reader_cache_info(&self) -> ReaderCacheInfo {
614        self.decoded_cache
615            .lock()
616            .unwrap_or_else(std::sync::PoisonError::into_inner)
617            .info()
618    }
619
620    /// Enumerates image resources and storage streams. Selection is never implicit.
621    pub fn streams(&self) -> Result<Vec<StreamInfo>> {
622        self.graph
623            .keys()
624            .filter(|id| {
625                [
626                    "Image",
627                    "DiskImage",
628                    "ImageStream",
629                    "Map",
630                    "FileImage",
631                    "FileSubStream",
632                ]
633                .iter()
634                .any(|t| self.has_type(id, t))
635            })
636            .map(|id| {
637                Ok(StreamInfo {
638                    id: id.clone(),
639                    types: self
640                        .properties(id)
641                        .filter(|p| p.predicate == RDF_TYPE)
642                        .map(|p| p.value.clone())
643                        .collect(),
644                    size: self.stream_size(id, &mut Vec::new()).ok(),
645                })
646            })
647            .collect()
648    }
649
650    /// Returns the size of a selected resource, resolving its dataStream reference.
651    pub fn size(&self, id: &str) -> Result<u64> {
652        self.stream_size(id, &mut Vec::new())
653    }
654
655    /// Reads a selected stream at a byte offset. Missing targets, cycles,
656    /// corrupt chunks, and unknown/unreadable symbolic data return errors.
657    pub fn read_at(&mut self, id: &str, buffer: &mut [u8], offset: u64) -> Result<usize> {
658        self.read_at_impl(id, buffer, offset, true)
659    }
660
661    /// Reads a resource sequentially. ZIP segments, including those behind
662    /// identity maps, retain one decompressor across reads; other resources
663    /// use positioned reads through their maps.
664    /// The caller must read the declared size to detect a truncated stream.
665    pub fn sequential_reader(&mut self, id: &str) -> Result<Box<dyn Read + '_>> {
666        let size = self.size(id)?;
667        self.read_inner(id, &mut [], 0, &mut Vec::new(), false)?;
668        let target = self.linear_backing(id)?;
669        if self.has_type(&target, "ZipSegment") || self.has_type(&target, "zip_segment") {
670            let path = self.path(&target)?;
671            let file = self.archive.by_name(&path)?;
672            if file.size() != size {
673                return Err(malformed("ZIP segment size mismatch"));
674            }
675            return Ok(Box::new(file));
676        }
677        Ok(Box::new(PositionedReader {
678            container: self,
679            id: id.to_owned(),
680            position: 0,
681        }))
682    }
683
684    fn read_at_uncached(&mut self, id: &str, buffer: &mut [u8], offset: u64) -> Result<usize> {
685        self.read_at_impl(id, buffer, offset, false)
686    }
687
688    fn read_at_impl(
689        &mut self,
690        id: &str,
691        buffer: &mut [u8],
692        offset: u64,
693        use_decoded_cache: bool,
694    ) -> Result<usize> {
695        let size = self.size(id)?;
696        if offset >= size || buffer.is_empty() {
697            return Ok(0);
698        }
699        let length = (size - offset).min(buffer.len() as u64) as usize;
700        self.read_inner(
701            id,
702            &mut buffer[..length],
703            offset,
704            &mut Vec::new(),
705            use_decoded_cache,
706        )?;
707        Ok(length)
708    }
709
710    /// Computes linear hashes with progress and cancellation. Does not verify
711    /// block hashes, map hashes, signatures, or metadata integrity.
712    pub fn verify(
713        &mut self,
714        id: &str,
715        progress: impl FnMut(u64, u64) -> ControlFlow<()>,
716    ) -> Result<Verification> {
717        self.copy_verified(id, &mut std::io::sink(), progress)
718    }
719
720    /// Copies exact logical bytes while computing and comparing linear hashes.
721    /// The caller owns staging/publication and must inspect `references_match`.
722    /// Partial output may remain in the sink after any failure or cancellation.
723    pub fn copy_verified(
724        &mut self,
725        id: &str,
726        output: &mut impl Write,
727        progress: impl FnMut(u64, u64) -> ControlFlow<()>,
728    ) -> Result<Verification> {
729        self.copy_verified_impl(id, output, None, progress)
730            .map(|(verification, _)| verification)
731    }
732
733    fn verify_with_paired_blocks(
734        &mut self,
735        id: &str,
736        block_size: usize,
737        progress: impl FnMut(u64, u64) -> ControlFlow<()>,
738    ) -> Result<(Verification, PairedBlockDigests)> {
739        let (verification, blocks) =
740            self.copy_verified_impl(id, &mut std::io::sink(), Some(block_size), progress)?;
741        Ok((
742            verification,
743            blocks.expect("paired block hashing requested"),
744        ))
745    }
746
747    fn copy_verified_impl(
748        &mut self,
749        id: &str,
750        output: &mut impl Write,
751        block_size: Option<usize>,
752        mut progress: impl FnMut(u64, u64) -> ControlFlow<()>,
753    ) -> Result<(Verification, Option<PairedBlockDigests>)> {
754        let size = self.size(id)?;
755        if size > self.limits.verification_bytes {
756            return Err(malformed("verification byte limit exceeded"));
757        }
758        self.cache = None;
759        self.index_cache = None;
760        self.decoded_cache
761            .lock()
762            .map_err(|_| malformed("AFF4 decoded cache lock poisoned"))?
763            .clear();
764        self.maps.clear();
765        self.read_inner(id, &mut [], 0, &mut Vec::new(), false)?;
766        let mut md5 = Md5::new();
767        let mut sha1 = Sha1::new();
768        let mut sha256 = Sha256::new();
769        let mut sha512 = sha2::Sha512::new();
770        let mut blake2b = <blake2::Blake2b512 as blake2::Digest>::new();
771        let mut blocks = block_size.map(PairedBlockHasher::new);
772        let mut write_error = None;
773        let walked = self.walk_bytes(id, |bytes, done| {
774            if let Err(error) = output.write_all(bytes) {
775                write_error = Some(error);
776                return ControlFlow::Break(());
777            }
778            md5.update(bytes);
779            sha1.update(bytes);
780            sha256.update(bytes);
781            sha512.update(bytes);
782            blake2::Digest::update(&mut blake2b, bytes);
783            if let Some(blocks) = &mut blocks {
784                blocks.update(bytes);
785            }
786            progress(done, size)
787        });
788        if let Some(error) = write_error {
789            return Err(error.into());
790        }
791        walked?;
792        let result = Verification {
793            bytes_verified: size,
794            md5: hex(&md5.finalize()),
795            sha1: hex(&sha1.finalize()),
796            sha256: hex(&sha256.finalize()),
797            sha512: hex(&sha512.finalize()),
798            blake2b: hex(&blake2::Digest::finalize(blake2b)),
799            references_match: None,
800            unsupported_hashes: Vec::new(),
801        };
802        let result = self.with_linear_references(id, result)?;
803        Ok((result, blocks.map(PairedBlockHasher::finish)))
804    }
805
806    fn with_linear_references(&self, id: &str, mut result: Verification) -> Result<Verification> {
807        result.references_match = None;
808        result.unsupported_hashes.clear();
809        for reference in self
810            .properties(id)
811            .filter(|p| is_property(&p.predicate, "hash"))
812        {
813            let datatype = reference.datatype.as_deref().unwrap_or("");
814            let expected = match datatype.strip_prefix(NS) {
815                Some("MD5") => &result.md5,
816                Some("SHA1") => &result.sha1,
817                Some("SHA256") => &result.sha256,
818                Some("SHA512") => &result.sha512,
819                Some("Blake2b" | "blake2b") => &result.blake2b,
820                _ => {
821                    result.unsupported_hashes.push(datatype.into());
822                    continue;
823                }
824            };
825            if reference.value.len() != expected.len()
826                || !reference.value.bytes().all(|b| b.is_ascii_hexdigit())
827            {
828                return Err(malformed("invalid linear hash"));
829            }
830            result.references_match = Some(
831                result.references_match.unwrap_or(true)
832                    && reference.value.eq_ignore_ascii_case(expected),
833            );
834        }
835        Ok(result)
836    }
837
838    // A dataStream wrapper reads the same bytes as its target. A Map is also
839    // equivalent only when one range covers the entire equally sized target
840    // from offset zero. Sparse, partial, and reordered maps retain their own
841    // traversal in full-container verification.
842    fn linear_backing(&mut self, id: &str) -> Result<String> {
843        let mut target = id.to_owned();
844        let mut visited = Vec::new();
845        loop {
846            enter(&target, &mut visited)?;
847            if self.inline_data(&target)?.is_some() {
848                return Ok(target);
849            }
850            if let Some(next) = self.value(&target, "dataStream")? {
851                target = next;
852                continue;
853            }
854            if self.has_type(&target, "Map") {
855                self.load_map(&target)?;
856                let map = &self.maps[&target];
857                if let [range] = map.ranges.as_slice()
858                    && range.start == 0
859                    && range.offset == 0
860                    && range.end == self.size(&target)?
861                    && self.size(&range.target).ok() == Some(range.end)
862                {
863                    target = range.target.clone();
864                    continue;
865                }
866            }
867            return Ok(target);
868        }
869    }
870
871    // ZIP files must be decoded once for sequential verification. Positioned
872    // reads remain available, but restarting Deflate for each buffer is quadratic.
873    fn walk_bytes(
874        &mut self,
875        id: &str,
876        mut consume: impl FnMut(&[u8], u64) -> ControlFlow<()>,
877    ) -> Result<()> {
878        let size = self.size(id)?;
879        if consume(&[], 0).is_break() {
880            return Err(Error::Aborted);
881        }
882        self.read_inner(id, &mut [], 0, &mut Vec::new(), false)?;
883        let target = self.linear_backing(id)?;
884        let mut buffer = vec![0; 1024 * 1024];
885        if self.has_type(&target, "ZipSegment") || self.has_type(&target, "zip_segment") {
886            let path = self.path(&target)?;
887            let mut file = self.archive.by_name(&path)?;
888            if file.size() != size {
889                return Err(malformed("ZIP segment size mismatch"));
890            }
891            let mut offset = 0;
892            while offset < size {
893                let take = (size - offset).min(buffer.len() as u64) as usize;
894                file.read_exact(&mut buffer[..take])?;
895                offset += take as u64;
896                if consume(&buffer[..take], offset).is_break() {
897                    return Err(Error::Aborted);
898                }
899            }
900            if file.read(&mut buffer[..1])? != 0 {
901                return Err(malformed("oversized ZIP segment"));
902            }
903        } else {
904            let mut offset = 0;
905            while offset < size {
906                let read = self.read_at_uncached(id, &mut buffer, offset)?;
907                if read == 0 {
908                    return Err(malformed("truncated stream"));
909                }
910                offset += read as u64;
911                if consume(&buffer[..read], offset).is_break() {
912                    return Err(Error::Aborted);
913                }
914            }
915        }
916        Ok(())
917    }
918
919    fn properties<'a>(&'a self, id: &str) -> impl Iterator<Item = &'a Property> {
920        self.graph.get(id).into_iter().flatten()
921    }
922    fn has_type(&self, id: &str, kind: &str) -> bool {
923        self.properties(id)
924            .any(|p| p.predicate == RDF_TYPE && is_property(&p.value, kind))
925    }
926    fn value(&self, id: &str, key: &str) -> Result<Option<String>> {
927        let values: BTreeSet<_> = self
928            .properties(id)
929            .filter(|p| is_property(&p.predicate, key))
930            .map(|p| p.value.clone())
931            .collect();
932        if values.len() > 1 {
933            return Err(malformed(format!("ambiguous {key}")));
934        }
935        Ok(values.into_iter().next())
936    }
937    fn number(&self, id: &str, key: &str) -> Result<u64> {
938        self.value(id, key)?
939            .ok_or_else(|| malformed(format!("missing {key}")))?
940            .parse()
941            .map_err(|_| malformed(format!("invalid {key}")))
942    }
943    fn stream_size(&self, id: &str, visited: &mut Vec<String>) -> Result<u64> {
944        enter(id, visited)?;
945        if self.inline_data(id)?.is_some() {
946            return self.number(id, "size");
947        }
948        if let Some(target) = self.value(id, "dataStream")? {
949            let size = self.stream_size(&target, visited)?;
950            if self.value(id, "size")?.is_some() && self.number(id, "size")? != size {
951                return Err(malformed("inconsistent stream size"));
952            }
953            Ok(size)
954        } else {
955            self.number(id, "size")
956        }
957    }
958    fn path(&self, id: &str) -> Result<String> {
959        if !self.has_type(id, "FileImage")
960            && let Some(path) = self.value(id, "fileName")?
961        {
962            return Ok(path);
963        }
964        storage_name(&self.archive, &self.volume, id, self.version)
965    }
966
967    fn inline_data(&self, id: &str) -> Result<Option<Vec<u8>>> {
968        let mut result = None;
969        for p in self.properties(id).filter(|p| {
970            (is_property(&p.predicate, "dataStream") || is_property(&p.predicate, "dataSteam"))
971                && p.datatype.as_deref() == Some(BASE64)
972        }) {
973            if p.value.len() > 1400 {
974                return Err(malformed("inline data exceeds 1 KiB limit"));
975            }
976            let bytes = base64::engine::general_purpose::STANDARD
977                .decode(&p.value)
978                .map_err(|_| malformed("invalid inline base64"))?;
979            if bytes.len() > 1024 || bytes.len() as u64 != self.number(id, "size")? {
980                return Err(malformed("inline data size mismatch"));
981            }
982            if result.as_ref().is_some_and(|old| old != &bytes) {
983                return Err(malformed("conflicting inline data"));
984            }
985            result = Some(bytes);
986        }
987        if result.is_some()
988            && self.properties(id).any(|p| {
989                (is_property(&p.predicate, "dataStream") || is_property(&p.predicate, "dataSteam"))
990                    && p.datatype.as_deref() != Some(BASE64)
991            })
992        {
993            return Err(malformed("conflicting inline and referenced data streams"));
994        }
995        Ok(result)
996    }
997    fn load_map(&mut self, id: &str) -> Result<()> {
998        if self.maps.contains_key(id) {
999            return Ok(());
1000        }
1001        let path = self.path(id)?;
1002        let data = member(
1003            &mut self.archive,
1004            &format!("{path}/map"),
1005            self.limits.member_bytes,
1006        )?;
1007        let index = member(
1008            &mut self.archive,
1009            &format!("{path}/idx"),
1010            self.limits.metadata_bytes,
1011        )?;
1012        let index =
1013            std::str::from_utf8(&index).map_err(|_| malformed("map target index is not UTF-8"))?;
1014        let targets: Vec<_> = index.lines().collect();
1015        if data.len() % 28 != 0 {
1016            return Err(malformed("partial map record"));
1017        }
1018        let records = data.len() / 28;
1019        if records > self.limits.map_bytes / std::mem::size_of::<Range>() {
1020            return Err(malformed("retained map byte limit exceeded"));
1021        }
1022        let mut ranges = Vec::new();
1023        ranges
1024            .try_reserve_exact(records)
1025            .map_err(allocation_error)?;
1026        let mut retained = 0_usize;
1027        let mut previous = 0;
1028        let size = self.number(id, "size")?;
1029        for record in data.as_chunks::<28>().0 {
1030            let start = u64::from_le_bytes(record[..8].try_into().unwrap());
1031            let length = u64::from_le_bytes(record[8..16].try_into().unwrap());
1032            let offset = u64::from_le_bytes(record[16..24].try_into().unwrap());
1033            let target = u32::from_le_bytes(record[24..].try_into().unwrap()) as usize;
1034            let end = start
1035                .checked_add(length)
1036                .ok_or_else(|| malformed("map overflow"))?;
1037            if length == 0 || start < previous || end > size || offset.checked_add(length).is_none()
1038            {
1039                return Err(malformed("invalid or overlapping map range"));
1040            }
1041            let target = *targets
1042                .get(target)
1043                .ok_or_else(|| malformed("missing map target"))?;
1044            retained = retained
1045                .checked_add(std::mem::size_of::<Range>())
1046                .and_then(|n| n.checked_add(target.len()))
1047                .filter(|n| *n <= self.limits.map_bytes)
1048                .ok_or_else(|| malformed("retained map byte limit exceeded"))?;
1049            ranges.push(Range {
1050                start,
1051                end,
1052                offset,
1053                target: target.to_owned(),
1054            });
1055            previous = end;
1056        }
1057        // Keep one map at a time, bounding retained range metadata across streams.
1058        self.maps.clear();
1059        self.maps.insert(
1060            id.into(),
1061            Arc::new(Map {
1062                ranges,
1063                gap: self
1064                    .value(id, "mapGapDefaultStream")?
1065                    .unwrap_or_else(|| format!("{NS}Zero")),
1066            }),
1067        );
1068        Ok(())
1069    }
1070
1071    fn read_stored_member_range(
1072        &mut self,
1073        name: &str,
1074        offset: u64,
1075        buffer: &mut [u8],
1076    ) -> Result<bool> {
1077        if self
1078            .stored_member
1079            .as_ref()
1080            .is_none_or(|member| member.name != name)
1081        {
1082            let directory_start = self.archive.central_directory_start();
1083            let file = self.archive.by_name(name)?;
1084            if file.size() > self.limits.member_bytes {
1085                return Err(malformed(format!("member exceeds resource limit: {name}")));
1086            }
1087            let size = file.size();
1088            let data_start = (file.compression() == CompressionMethod::Stored
1089                && !file.encrypted()
1090                && file.compressed_size() == size)
1091                .then(|| file.data_start())
1092                .flatten();
1093            if data_start
1094                .and_then(|start| start.checked_add(size))
1095                .is_some_and(|end| end > directory_start)
1096            {
1097                return Err(malformed("stored ZIP member exceeds archive data"));
1098            }
1099            self.stored_member = Some(StoredMember {
1100                name: name.to_owned(),
1101                size,
1102                data_start,
1103            });
1104        }
1105        let member = self.stored_member.as_ref().unwrap();
1106        if offset
1107            .checked_add(buffer.len() as u64)
1108            .is_none_or(|end| end > member.size)
1109        {
1110            return Err(malformed("member range exceeds stored ZIP entry"));
1111        }
1112        let Some(data_start) = member.data_start else {
1113            return Ok(false);
1114        };
1115        let absolute = data_start
1116            .checked_add(offset)
1117            .ok_or_else(|| malformed("stored ZIP member offset overflow"))?;
1118        read_file_exact_at(&self.stored_source, absolute, buffer)?;
1119        self.decoded_cache
1120            .lock()
1121            .map_err(|_| malformed("AFF4 decoded cache lock poisoned"))?
1122            .record_stored_member_range_read(buffer.len());
1123        Ok(true)
1124    }
1125
1126    fn read_inner(
1127        &mut self,
1128        id: &str,
1129        buffer: &mut [u8],
1130        offset: u64,
1131        visited: &mut Vec<String>,
1132        use_decoded_cache: bool,
1133    ) -> Result<()> {
1134        enter(id, visited)?;
1135        let result = self.read_inner_impl(id, buffer, offset, visited, use_decoded_cache);
1136        visited.pop();
1137        result
1138    }
1139    fn read_inner_impl(
1140        &mut self,
1141        id: &str,
1142        buffer: &mut [u8],
1143        offset: u64,
1144        visited: &mut Vec<String>,
1145        use_decoded_cache: bool,
1146    ) -> Result<()> {
1147        if id == format!("{NS}Zero") {
1148            buffer.fill(0);
1149            return Ok(());
1150        }
1151        if let Some(value) = id.strip_prefix(&format!("{NS}SymbolicStream")) {
1152            if value.len() != 2 {
1153                return Err(malformed("invalid symbolic byte"));
1154            }
1155            buffer.fill(
1156                u8::from_str_radix(value, 16).map_err(|_| malformed("invalid symbolic byte"))?,
1157            );
1158            return Ok(());
1159        }
1160        if id == format!("{NS}UnknownData") || id == format!("{NS}UnreadableData") {
1161            return Err(Error::Unsupported(
1162                "unknown or unreadable evidence range".into(),
1163            ));
1164        }
1165        let size = self.size(id)?;
1166        if offset
1167            .checked_add(buffer.len() as u64)
1168            .is_none_or(|end| end > size)
1169        {
1170            return Err(malformed("target range exceeds stream"));
1171        }
1172        if let Some(data) = self.inline_data(id)? {
1173            buffer.copy_from_slice(&data[offset as usize..offset as usize + buffer.len()]);
1174            return Ok(());
1175        }
1176        if let Some(target) = self.value(id, "dataStream")? {
1177            if self.has_type(id, "ContiguousImage") && self.has_type(&target, "Map") {
1178                self.load_map(&target)?;
1179                let map = &self.maps[&target];
1180                let mut end = 0;
1181                for range in &map.ranges {
1182                    if range.start != end {
1183                        return Err(malformed("gap in declared contiguous image"));
1184                    }
1185                    end = range.end;
1186                }
1187                if end != size {
1188                    return Err(malformed("truncated contiguous image map"));
1189                }
1190            }
1191            return self.read_inner(&target, buffer, offset, visited, use_decoded_cache);
1192        }
1193        if self.has_type(id, "ZipSegment") || self.has_type(id, "zip_segment") {
1194            let path = self.path(id)?;
1195            let mut file = self.archive.by_name(&path)?;
1196            if file.size() != size {
1197                return Err(malformed("ZIP segment size mismatch"));
1198            }
1199            if std::io::copy(&mut file.by_ref().take(offset), &mut std::io::sink())? != offset {
1200                return Err(malformed("truncated ZIP segment"));
1201            }
1202            file.read_exact(buffer)?;
1203            if offset + buffer.len() as u64 == size {
1204                let mut eof = [0];
1205                if file.read(&mut eof)? != 0 {
1206                    return Err(malformed("oversized ZIP segment"));
1207                }
1208            }
1209            return Ok(());
1210        }
1211        if self.has_type(id, "Map") {
1212            self.load_map(id)?;
1213            let map = self.maps[id].clone();
1214            let mut done = 0;
1215            while done < buffer.len() {
1216                let position = offset + done as u64;
1217                let next = map.ranges.partition_point(|r| r.end <= position);
1218                let (target, target_offset, end) = match map.ranges.get(next) {
1219                    Some(range) if range.start <= position => (
1220                        range.target.as_str(),
1221                        range.offset + (position - range.start),
1222                        range.end,
1223                    ),
1224                    next => (
1225                        map.gap.as_str(),
1226                        position,
1227                        next.map_or(self.number(id, "size")?, |r| r.start),
1228                    ),
1229                };
1230                let length = (end - position).min((buffer.len() - done) as u64) as usize;
1231                if length == 0 {
1232                    return Err(malformed("map makes no progress"));
1233                }
1234                self.read_inner(
1235                    target,
1236                    &mut buffer[done..done + length],
1237                    target_offset,
1238                    visited,
1239                    use_decoded_cache,
1240                )?;
1241                done += length;
1242            }
1243            return Ok(());
1244        }
1245        if !self.has_type(id, "ImageStream") {
1246            return Err(Error::Unsupported(format!("stream kind: {id}")));
1247        }
1248        let chunk_size = self.number(id, "chunkSize")?;
1249        let per_bevy = self.number(id, "chunksInSegment")?;
1250        if chunk_size == 0 || chunk_size > self.limits.chunk_bytes || per_bevy == 0 {
1251            return Err(malformed("invalid chunk geometry"));
1252        }
1253        let path = self.path(id)?;
1254        let compression = self.value(id, "compressionMethod")?;
1255        let mut done = 0;
1256        while done < buffer.len() {
1257            let position = offset + done as u64;
1258            let chunk = position / chunk_size;
1259            let cache_key = DecodedCacheKey::Chunk {
1260                volume: self.volume.clone(),
1261                resource: id.to_owned(),
1262                chunk,
1263            };
1264            let cached = if use_decoded_cache {
1265                self.decoded_cache
1266                    .lock()
1267                    .map_err(|_| malformed("AFF4 decoded cache lock poisoned"))?
1268                    .get_chunk(&cache_key)
1269            } else {
1270                None
1271            };
1272            if let Some(decoded) = cached {
1273                let begin = (position % chunk_size) as usize;
1274                let take = (decoded.len() - begin).min(buffer.len() - done);
1275                buffer[done..done + take].copy_from_slice(&decoded[begin..begin + take]);
1276                done += take;
1277                continue;
1278            }
1279            let name = format!("{path}/{:08}", chunk / per_bevy);
1280            if self
1281                .index_cache
1282                .as_ref()
1283                .is_none_or(|(key, _)| key != &name)
1284            {
1285                self.index_cache = Some((
1286                    name.clone(),
1287                    member(
1288                        &mut self.archive,
1289                        &format!("{name}.index"),
1290                        self.limits.member_bytes,
1291                    )?,
1292                ));
1293            }
1294            let index = &self.index_cache.as_ref().unwrap().1;
1295            if !index.len().is_multiple_of(12) {
1296                return Err(malformed("partial chunk index record"));
1297            }
1298            let index_offset = usize::try_from(chunk % per_bevy)
1299                .ok()
1300                .and_then(|n| n.checked_mul(12))
1301                .ok_or_else(|| malformed("index overflow"))?;
1302            let index_end = index_offset
1303                .checked_add(12)
1304                .ok_or_else(|| malformed("index overflow"))?;
1305            let record = index
1306                .get(index_offset..index_end)
1307                .ok_or_else(|| malformed("missing chunk index"))?;
1308            let start = u64::from_le_bytes(record[..8].try_into().unwrap());
1309            let length = u32::from_le_bytes(record[8..].try_into().unwrap()) as u64;
1310            let end = start
1311                .checked_add(length)
1312                .filter(|end| *end <= self.limits.member_bytes)
1313                .ok_or_else(|| malformed("chunk outside bevy"))?;
1314            let mut direct = chunk_buffer(length as usize)?;
1315            let direct_read = self.read_stored_member_range(&name, start, &mut direct)?;
1316            if direct_read {
1317                self.cache = None;
1318            } else if self.cache.as_ref().is_none_or(|(key, _)| key != &name) {
1319                self.cache = Some((
1320                    name.clone(),
1321                    member(&mut self.archive, &name, self.limits.member_bytes)?,
1322                ));
1323            }
1324            let encoded = if direct_read {
1325                direct.as_slice()
1326            } else {
1327                let data = &self.cache.as_ref().unwrap().1;
1328                if end > data.len() as u64 {
1329                    return Err(malformed("chunk outside bevy"));
1330                }
1331                &data[start as usize..end as usize]
1332            };
1333            let decode_started = Instant::now();
1334            let decoded = if length == chunk_size {
1335                if direct_read {
1336                    direct
1337                } else {
1338                    encoded.to_vec()
1339                }
1340            } else {
1341                decode(encoded, compression.as_deref(), chunk_size as usize)?
1342            };
1343            self.decoded_cache
1344                .lock()
1345                .map_err(|_| malformed("AFF4 decoded cache lock poisoned"))?
1346                .record_decode(decoded.len(), decode_started.elapsed());
1347            if decoded.len() != chunk_size as usize {
1348                return Err(malformed("decoded chunk length mismatch"));
1349            }
1350            let begin = (position % chunk_size) as usize;
1351            let take = (decoded.len() - begin).min(buffer.len() - done);
1352            buffer[done..done + take].copy_from_slice(&decoded[begin..begin + take]);
1353            done += take;
1354            if use_decoded_cache {
1355                self.decoded_cache
1356                    .lock()
1357                    .map_err(|_| malformed("AFF4 decoded cache lock poisoned"))?
1358                    .insert(cache_key, Arc::new(decoded));
1359            }
1360        }
1361        Ok(())
1362    }
1363}
1364
1365#[derive(Clone)]
1366pub(super) struct PairedBlockDigests {
1367    md5: Vec<[u8; 16]>,
1368    sha256: Vec<[u8; 32]>,
1369}
1370
1371struct PairedBlockHasher {
1372    block_size: usize,
1373    used: usize,
1374    md5: Md5,
1375    sha256: Sha256,
1376    result: PairedBlockDigests,
1377}
1378
1379impl PairedBlockHasher {
1380    fn new(block_size: usize) -> Self {
1381        debug_assert!(block_size > 0);
1382        Self {
1383            block_size,
1384            used: 0,
1385            md5: Md5::new(),
1386            sha256: Sha256::new(),
1387            result: PairedBlockDigests {
1388                md5: Vec::new(),
1389                sha256: Vec::new(),
1390            },
1391        }
1392    }
1393
1394    fn update(&mut self, mut bytes: &[u8]) {
1395        while !bytes.is_empty() {
1396            let take = bytes.len().min(self.block_size - self.used);
1397            self.md5.update(&bytes[..take]);
1398            self.sha256.update(&bytes[..take]);
1399            self.used += take;
1400            bytes = &bytes[take..];
1401            if self.used == self.block_size {
1402                self.finish_block();
1403            }
1404        }
1405    }
1406
1407    fn finish(mut self) -> PairedBlockDigests {
1408        if self.used != 0 {
1409            self.finish_block();
1410        }
1411        self.result
1412    }
1413
1414    fn finish_block(&mut self) {
1415        self.result.md5.push(
1416            std::mem::replace(&mut self.md5, Md5::new())
1417                .finalize()
1418                .into(),
1419        );
1420        self.result.sha256.push(
1421            std::mem::replace(&mut self.sha256, Sha256::new())
1422                .finalize()
1423                .into(),
1424        );
1425        self.used = 0;
1426    }
1427}
1428
1429#[cfg(unix)]
1430fn read_file_exact_at(file: &File, offset: u64, buffer: &mut [u8]) -> std::io::Result<()> {
1431    use std::os::unix::fs::FileExt;
1432    file.read_exact_at(buffer, offset)
1433}
1434
1435#[cfg(windows)]
1436fn read_file_exact_at(file: &File, offset: u64, mut buffer: &mut [u8]) -> std::io::Result<()> {
1437    use std::os::windows::fs::FileExt;
1438    let mut position = offset;
1439    while !buffer.is_empty() {
1440        let read = file.seek_read(buffer, position)?;
1441        if read == 0 {
1442            return Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof));
1443        }
1444        position = position
1445            .checked_add(read as u64)
1446            .ok_or_else(|| std::io::Error::from(std::io::ErrorKind::InvalidInput))?;
1447        buffer = &mut buffer[read..];
1448    }
1449    Ok(())
1450}
1451
1452#[cfg(not(any(unix, windows)))]
1453fn read_file_exact_at(file: &File, offset: u64, buffer: &mut [u8]) -> std::io::Result<()> {
1454    use std::io::{Seek, SeekFrom};
1455    let mut file = file.try_clone()?;
1456    file.seek(SeekFrom::Start(offset))?;
1457    file.read_exact(buffer)
1458}
1459
1460fn enter(id: &str, visited: &mut Vec<String>) -> Result<()> {
1461    if visited.len() >= 32 || visited.iter().any(|v| v == id) {
1462        return Err(malformed("cyclic or excessively deep stream references"));
1463    }
1464    visited.push(id.to_owned());
1465    Ok(())
1466}
1467
1468fn is_property(predicate: &str, local: &str) -> bool {
1469    [NS, LOGICAL_NS, LEGACY_LOGICAL_NS]
1470        .iter()
1471        .any(|ns| predicate.strip_prefix(ns) == Some(local))
1472}
1473
1474fn storage_name(
1475    archive: &ZipArchive<File>,
1476    volume: &str,
1477    id: &str,
1478    version: (u32, u32),
1479) -> Result<String> {
1480    if version != (1, 0) && archive.index_for_name(id).is_some() {
1481        return Ok(id.to_owned());
1482    }
1483    if let Some(path) = id.strip_prefix(&format!("{volume}/")) {
1484        return Ok(if version == (1, 0) {
1485            escape(path)
1486        } else {
1487            path.to_owned()
1488        });
1489    }
1490    let rest = id
1491        .strip_prefix("aff4://")
1492        .ok_or_else(|| Error::Unsupported("non-AFF4 storage reference".into()))?;
1493    // Legacy AFF4-L producers also use the physical ARN escaping for chunked
1494    // storage. Resolve only an existing prefix; draft 2.1 uses unescaped ARNs.
1495    let legacy = format!("aff4%3A%2F%2F{}", escape(rest));
1496    if version == (1, 1)
1497        && archive
1498            .file_names()
1499            .any(|n| n.starts_with(&format!("{legacy}/")))
1500    {
1501        return Ok(legacy);
1502    }
1503    Ok(if version == (1, 0) {
1504        format!("aff4%3A%2F%2F{}", escape(rest))
1505    } else {
1506        id.to_owned()
1507    })
1508}
1509fn member(archive: &mut ZipArchive<File>, name: &str, limit: u64) -> Result<Vec<u8>> {
1510    let file = archive.by_name(name)?;
1511    if file.size() > limit {
1512        return Err(malformed(format!("member exceeds resource limit: {name}")));
1513    }
1514    // Grow from bytes actually decoded, not an untrusted advertised size.
1515    read_buffer(file, limit)
1516}
1517
1518fn allocation_error(error: std::collections::TryReserveError) -> Error {
1519    std::io::Error::new(std::io::ErrorKind::OutOfMemory, error).into()
1520}
1521
1522fn read_buffer(mut reader: impl Read, limit: u64) -> Result<Vec<u8>> {
1523    let mut data = Vec::new();
1524    let mut buffer = [0; 64 * 1024];
1525    loop {
1526        let take = (limit.saturating_sub(data.len() as u64).saturating_add(1))
1527            .min(buffer.len() as u64) as usize;
1528        let count = match reader.read(&mut buffer[..take]) {
1529            Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,
1530            value => value?,
1531        };
1532        if count == 0 {
1533            return Ok(data);
1534        }
1535        if count as u64 > limit.saturating_sub(data.len() as u64) {
1536            return Err(malformed("decoded member exceeds limit"));
1537        }
1538        data.try_reserve(count).map_err(allocation_error)?;
1539        data.extend_from_slice(&buffer[..count]);
1540    }
1541}
1542fn decode(data: &[u8], method: Option<&str>, size: usize) -> Result<Vec<u8>> {
1543    match method {
1544        Some("http://code.google.com/p/snappy/") => {
1545            if snap::raw::decompress_len(data).map_err(|e| malformed(e.to_string()))? != size {
1546                return Err(malformed("snappy size mismatch"));
1547            }
1548            let mut decoded = chunk_buffer(size)?;
1549            let count = snap::raw::Decoder::new()
1550                .decompress(data, &mut decoded)
1551                .map_err(|e| malformed(e.to_string()))?;
1552            decoded.truncate(count);
1553            Ok(decoded)
1554        }
1555        Some("https://www.ietf.org/rfc/rfc1950.txt") => {
1556            read_buffer(flate2::read::ZlibDecoder::new(data), size as u64)
1557        }
1558        Some("https://tools.ietf.org/html/rfc1951") => {
1559            read_buffer(flate2::read::DeflateDecoder::new(data), size as u64)
1560        }
1561        Some("https://code.google.com/p/lz4/") => {
1562            let mut decoded = chunk_buffer(size)?;
1563            let count = lz4_flex::block::decompress_into(data, &mut decoded)
1564                .map_err(|e| malformed(e.to_string()))?;
1565            decoded.truncate(count);
1566            Ok(decoded)
1567        }
1568        _ => Err(Error::Unsupported("chunk compression method".into())),
1569    }
1570}
1571
1572fn chunk_buffer(size: usize) -> Result<Vec<u8>> {
1573    let mut buffer = Vec::new();
1574    buffer.try_reserve_exact(size).map_err(allocation_error)?;
1575    buffer.resize(size, 0);
1576    Ok(buffer)
1577}
1578pub(crate) fn hex(data: &[u8]) -> String {
1579    data.iter().map(|b| format!("{b:02x}")).collect()
1580}
1581fn escape(value: &str) -> String {
1582    let mut output = String::new();
1583    for b in value.bytes() {
1584        if b.is_ascii_alphanumeric() || b"-._~/".contains(&b) {
1585            output.push(char::from(b));
1586        } else {
1587            output.push_str(&format!("%{b:02X}"));
1588        }
1589    }
1590    output
1591}
1592
1593#[cfg(test)]
1594mod tests {
1595    use super::*;
1596
1597    fn chunk(chunk: u64) -> DecodedCacheKey {
1598        DecodedCacheKey::Chunk {
1599            volume: "aff4://volume".into(),
1600            resource: "aff4://volume/data".into(),
1601            chunk,
1602        }
1603    }
1604
1605    #[test]
1606    fn decoded_cache_is_byte_bounded_and_lru() {
1607        let mut cache = DecodedChunkCache::with_capacity(8);
1608        cache.insert(chunk(0), Arc::new(vec![0; 4]));
1609        cache.insert(chunk(1), Arc::new(vec![1; 4]));
1610        assert!(cache.get_chunk(&chunk(0)).is_some());
1611        cache.insert(chunk(2), Arc::new(vec![2; 4]));
1612
1613        assert!(cache.get_chunk(&chunk(0)).is_some());
1614        assert!(cache.get_chunk(&chunk(1)).is_none());
1615        assert!(cache.get_chunk(&chunk(2)).is_some());
1616        assert_eq!(cache.current_bytes, 8);
1617        assert_eq!(cache.statistics.decoded_cache_evictions, 1);
1618        assert_eq!(cache.info().capacity_bytes(), 8);
1619    }
1620
1621    #[test]
1622    fn oversized_entry_is_not_retained() {
1623        let mut cache = DecodedChunkCache::with_capacity(4);
1624        cache.insert(chunk(0), Arc::new(vec![0; 5]));
1625        assert_eq!(cache.current_bytes, 0);
1626        assert_eq!(cache.entries.len(), 0);
1627    }
1628}