Skip to main content

aff4_image/
volume_set.rs

1//! Explicit multi-volume physical reading. Graphs retain their source context.
2use super::*;
3use std::path::PathBuf;
4
5#[path = "set_verify.rs"]
6mod set_verify;
7pub use set_verify::{SetVerification, SetVolumeVerification};
8
9/// Attribution of a stored ImageStream to its owning ZIP volume.
10#[derive(Debug, Clone, serde::Serialize)]
11pub struct VolumeSource {
12    /// Data stream resource.
13    pub stream: String,
14    /// Owning volume identifier.
15    pub volume: String,
16    /// Supplied path to that volume.
17    pub path: PathBuf,
18}
19
20/// Whole-image digest over a resolved volume set, distinct from block-map hashes.
21#[derive(Debug, Clone, serde::Serialize)]
22pub struct SetDigest {
23    /// Image identifier.
24    pub image: String,
25    /// Logical bytes traversed.
26    pub bytes: u64,
27    /// SHA256 of the assembled logical image.
28    pub sha256: String,
29    /// Comparison with an independently supplied whole-image SHA256.
30    pub external_match: Option<bool>,
31    /// Stream owners, sorted by stream identifier.
32    pub sources: Vec<VolumeSource>,
33}
34
35/// A primary physical container plus explicitly supplied companion volumes.
36/// No filenames are guessed, no graphs are flattened, and ambiguous owners fail.
37/// This reader does not yet certify a striped image's composite block-map digest.
38pub struct VolumeSet {
39    volumes: Vec<Container>,
40    paths: Vec<PathBuf>,
41    owners: BTreeMap<String, usize>,
42    validated: BTreeMap<(usize, String), (Arc<Map>, u64)>,
43    cached_owner: Option<usize>,
44}
45
46impl VolumeSet {
47    /// Opens the primary and companions with a shared metadata/directory budget.
48    /// The primary must contain the selected image's complete Map. Named foreign
49    /// volume references must be supplied; duplicate owners/volume IDs are errors.
50    pub fn open(paths: &[PathBuf]) -> Result<Self> {
51        Self::open_with_limits(paths, Limits::default())
52    }
53
54    /// Opens at most 128 volumes. Metadata bytes, triples, ZIP directory bytes,
55    /// and entry counts are aggregate limits across the entire set. Chunk and
56    /// member limits apply per object; only one volume retains payload caches.
57    pub fn open_with_limits(paths: &[PathBuf], limits: Limits) -> Result<Self> {
58        if paths.is_empty() || paths.len() > 128 {
59            return Err(malformed("volume set must contain 1 through 128 paths"));
60        }
61        let mut volumes = Vec::new();
62        let mut ids = BTreeSet::new();
63        let mut remaining = limits.clone();
64        for path in paths {
65            let mut volume = Container::open_with_limits(path, remaining.clone())?;
66            remaining.metadata_bytes -= volume.usage.metadata_bytes;
67            remaining.triples -= volume.usage.triples;
68            remaining.directory_bytes -= volume.usage.directory_bytes;
69            remaining.archive_entries -= volume.usage.entries;
70            volume.limits = limits.clone();
71            if volume.version != (1, 0) {
72                return Err(Error::Unsupported(
73                    "volume sets currently support physical AFF4 1.0".into(),
74                ));
75            }
76            if !ids.insert(volume.volume.clone()) {
77                return Err(malformed("duplicate volume identifier"));
78            }
79            volumes.push(volume);
80        }
81        Self::from_containers(volumes, paths.to_vec())
82    }
83
84    pub(super) fn from_containers(
85        mut volumes: Vec<Container>,
86        paths: Vec<PathBuf>,
87    ) -> Result<Self> {
88        let ids: BTreeSet<_> = volumes.iter().map(|volume| volume.volume.clone()).collect();
89        if ids.len() != volumes.len() {
90            return Err(malformed("duplicate volume identifier"));
91        }
92        let mut owners = BTreeMap::new();
93        for (index, volume) in volumes.iter().enumerate() {
94            for id in volume
95                .graph
96                .keys()
97                .filter(|id| volume.has_type(id, "ImageStream"))
98            {
99                if let Some(stored) = volume.value(id, "stored")?
100                    && !ids.contains(&stored)
101                {
102                    return Err(malformed(format!("missing source volume {stored}")));
103                }
104                let Ok(size) = volume.number(id, "size") else {
105                    continue;
106                };
107                let path = volume.path(id)?;
108                let owns = if size == 0 {
109                    volume.value(id, "stored")?.as_deref() == Some(&volume.volume)
110                } else {
111                    volume
112                        .archive
113                        .index_for_name(&format!("{path}/00000000"))
114                        .is_some()
115                };
116                if owns {
117                    if volume
118                        .value(id, "stored")?
119                        .is_some_and(|v| v != volume.volume)
120                    {
121                        return Err(malformed("local stream conflicts with declared owner"));
122                    }
123                    if owners.insert(id.clone(), index).is_some() {
124                        return Err(malformed(format!("ambiguous stream owner: {id}")));
125                    }
126                }
127            }
128        }
129        // Validate supplied geometry against the owner's metadata. A foreign stub
130        // may omit it, but an explicit disagreement is not silently overridden.
131        for (id, &owner) in &owners {
132            for volume in &volumes {
133                if !volume.graph.contains_key(id) {
134                    continue;
135                }
136                for key in [
137                    "size",
138                    "chunkSize",
139                    "chunksInSegment",
140                    "compressionMethod",
141                    "stored",
142                ] {
143                    if let Some(value) = volume.value(id, key)? {
144                        let expected = if key == "stored" {
145                            Some(volumes[owner].volume.clone())
146                        } else {
147                            volumes[owner].value(id, key)?
148                        };
149                        if expected.as_ref() != Some(&value) {
150                            return Err(malformed(format!("conflicting {key} for {id}")));
151                        }
152                    }
153                }
154            }
155        }
156        let decoded_cache = Arc::new(Mutex::new(DecodedChunkCache::new()));
157        for volume in &mut volumes {
158            volume.decoded_cache = Arc::clone(&decoded_cache);
159        }
160        Ok(Self {
161            volumes,
162            paths,
163            owners,
164            validated: BTreeMap::new(),
165            cached_owner: None,
166        })
167    }
168
169    /// Identifiers of physical images described by the primary volume.
170    pub fn images(&self) -> Vec<String> {
171        self.volumes[0]
172            .graph
173            .keys()
174            .filter(|id| self.volumes[0].has_type(id, "DiskImage"))
175            .cloned()
176            .collect()
177    }
178
179    /// Data stream ownership; metadata itself remains scoped to each volume.
180    pub fn sources(&self) -> Vec<VolumeSource> {
181        self.owners
182            .iter()
183            .map(|(id, &n)| VolumeSource {
184                stream: id.clone(),
185                volume: self.volumes[n].volume.clone(),
186                path: self.paths[n].clone(),
187            })
188            .collect()
189    }
190
191    /// Returns cumulative payload-reader counters shared by all volumes.
192    pub fn reader_statistics(&self) -> ReaderStatistics {
193        self.volumes[0].reader_statistics()
194    }
195
196    /// Returns shared automatic decoded payload-cache usage.
197    pub fn reader_cache_info(&self) -> ReaderCacheInfo {
198        self.volumes[0].reader_cache_info()
199    }
200
201    /// Opens a primary map and validates all referenced storage before reading.
202    fn image_map(&mut self, id: &str) -> Result<(Arc<Map>, u64)> {
203        self.image_map_in(0, id)
204    }
205
206    fn image_map_in(&mut self, primary: usize, id: &str) -> Result<(Arc<Map>, u64)> {
207        if let Some(result) = self.validated.get(&(primary, id.to_owned())) {
208            return Ok(result.clone());
209        }
210        if !self.volumes[primary].has_type(id, "DiskImage") {
211            return Err(Error::Unsupported("select a primary DiskImage".into()));
212        }
213        let target = self.volumes[primary]
214            .value(id, "dataStream")?
215            .ok_or_else(|| malformed("image has no dataStream"))?;
216        self.volumes[primary].load_map(&target)?;
217        let map = self.volumes[primary].maps[&target].clone();
218        // The set retains one validated map, not one map per primary container.
219        self.volumes[primary].maps.clear();
220        let size = self.volumes[primary].number(&target, "size")?;
221        if self.volumes[primary].value(id, "size")?.is_some()
222            && self.volumes[primary].number(id, "size")? != size
223        {
224            return Err(malformed("image/map size mismatch"));
225        }
226        let contiguous = self.volumes[primary].has_type(id, "ContiguousImage");
227        let mut end = 0;
228        for range in &map.ranges {
229            if contiguous && range.start != end {
230                return Err(malformed("gap in contiguous image"));
231            }
232            end = range.end;
233            if is_symbolic(&range.target) {
234                continue;
235            }
236            let owner = *self
237                .owners
238                .get(&range.target)
239                .ok_or_else(|| malformed(format!("missing owner for {}", range.target)))?;
240            let size = self.volumes[owner].number(&range.target, "size")?;
241            if range
242                .offset
243                .checked_add(range.end - range.start)
244                .is_none_or(|end| end > size)
245            {
246                return Err(malformed("map exceeds owned stream"));
247            }
248        }
249        if contiguous && end != size {
250            return Err(malformed("truncated contiguous map"));
251        }
252        if !is_symbolic(&map.gap) && !self.owners.contains_key(&map.gap) {
253            return Err(Error::Unsupported("external/nested gap stream".into()));
254        }
255        self.validated.clear();
256        self.validated
257            .insert((primary, id.into()), (map.clone(), size));
258        Ok((map, size))
259    }
260
261    pub(super) fn equivalent_disk_maps(
262        &mut self,
263        left: usize,
264        right: usize,
265        id: &str,
266    ) -> Result<bool> {
267        let (left, size) = self.image_map_in(left, id)?;
268        let (right, other_size) = self.image_map_in(right, id)?;
269        if size != other_size {
270            return Ok(false);
271        }
272        let mut position = 0;
273        while position < size {
274            let (a, a_offset, a_end) = map_interval(&left, position, size);
275            let (b, b_offset, b_end) = map_interval(&right, position, size);
276            if a != b || (!is_symbolic(a) && a_offset != b_offset) {
277                return Ok(false);
278            }
279            position = a_end.min(b_end);
280        }
281        Ok(true)
282    }
283
284    /// Logical size of a selected primary image, after dependency validation.
285    pub fn size(&mut self, id: &str) -> Result<u64> {
286        self.image_map(id).map(|(_, size)| size)
287    }
288
289    /// Reads assembled image bytes using the primary Map and explicit owners.
290    /// Unknown/unreadable symbolic data and missing companions fail closed.
291    pub fn read_at(&mut self, id: &str, buffer: &mut [u8], offset: u64) -> Result<usize> {
292        self.read_disk_at(0, true, id, buffer, offset)
293    }
294
295    pub(super) fn disk_size(&mut self, primary: usize, mapped: bool, id: &str) -> Result<u64> {
296        if mapped {
297            self.image_map_in(primary, id).map(|(_, size)| size)
298        } else {
299            self.volumes[primary].size(id)
300        }
301    }
302
303    pub(super) fn read_disk_at(
304        &mut self,
305        primary: usize,
306        mapped: bool,
307        id: &str,
308        buffer: &mut [u8],
309        offset: u64,
310    ) -> Result<usize> {
311        let size = self.disk_size(primary, mapped, id)?;
312        if offset >= size || buffer.is_empty() {
313            return Ok(0);
314        }
315        let length = (size - offset).min(buffer.len() as u64) as usize;
316        if length >= LOGICAL_READ_AHEAD_BYTES {
317            let read =
318                self.read_disk_at_impl(primary, mapped, id, &mut buffer[..length], offset, false)?;
319            self.retain_complete_pages(primary, id, &buffer[..read], offset)?;
320            return Ok(read);
321        }
322
323        let mut done = 0;
324        while done < length {
325            let position = offset
326                .checked_add(done as u64)
327                .ok_or_else(|| malformed("AFF4 read offset overflow"))?;
328            let page_start =
329                position / LOGICAL_READ_AHEAD_BYTES as u64 * LOGICAL_READ_AHEAD_BYTES as u64;
330            let page_offset = usize::try_from(position - page_start)
331                .map_err(|_| malformed("AFF4 page offset overflow"))?;
332            let key = self.disk_page_key(primary, id, page_start)?;
333            let cached = self.volumes[primary]
334                .decoded_cache
335                .lock()
336                .map_err(|_| malformed("AFF4 decoded cache lock poisoned"))?
337                .get_page(&key);
338            let page = if let Some(cached) = cached {
339                Some(cached)
340            } else {
341                let page_length = LOGICAL_READ_AHEAD_BYTES
342                    .min(usize::try_from(size - page_start).unwrap_or(LOGICAL_READ_AHEAD_BYTES));
343                let mut page = vec![0; page_length];
344                match self.read_disk_at_impl(primary, mapped, id, &mut page, page_start, false) {
345                    Ok(read) if read == page_length => {
346                        let page = Arc::new(page);
347                        let mut cache = self.volumes[primary]
348                            .decoded_cache
349                            .lock()
350                            .map_err(|_| malformed("AFF4 decoded cache lock poisoned"))?;
351                        cache.record_read_ahead_prefetch();
352                        cache.insert(key, Arc::clone(&page));
353                        Some(page)
354                    }
355                    Ok(_) => return Err(malformed("truncated AFF4 disk page")),
356                    Err(_) => None,
357                }
358            };
359            let Some(page) = page else {
360                // Read-ahead must not make a valid requested range fail only
361                // because later bytes in the same page are unreadable.
362                let page_remaining = LOGICAL_READ_AHEAD_BYTES - page_offset;
363                let take = page_remaining.min(length - done);
364                let read = self.read_disk_at_impl(
365                    primary,
366                    mapped,
367                    id,
368                    &mut buffer[done..done + take],
369                    position,
370                    false,
371                )?;
372                if read != take {
373                    return Err(malformed("truncated AFF4 disk read"));
374                }
375                done += take;
376                continue;
377            };
378            let take = (page.len() - page_offset).min(length - done);
379            buffer[done..done + take].copy_from_slice(&page[page_offset..page_offset + take]);
380            done += take;
381        }
382        Ok(length)
383    }
384
385    fn disk_page_key(&self, primary: usize, id: &str, offset: u64) -> Result<DecodedCacheKey> {
386        let volume = self
387            .volumes
388            .get(primary)
389            .ok_or_else(|| malformed("AFF4 primary volume index out of range"))?
390            .volume
391            .clone();
392        Ok(DecodedCacheKey::DiskPage {
393            volume,
394            resource: id.to_owned(),
395            offset,
396        })
397    }
398
399    fn retain_complete_pages(
400        &self,
401        primary: usize,
402        id: &str,
403        buffer: &[u8],
404        offset: u64,
405    ) -> Result<()> {
406        let request_end = offset
407            .checked_add(buffer.len() as u64)
408            .ok_or_else(|| malformed("AFF4 read offset overflow"))?;
409        let page_bytes = LOGICAL_READ_AHEAD_BYTES as u64;
410        let mut page_start = match offset % page_bytes {
411            0 => offset,
412            remainder => offset
413                .checked_add(page_bytes - remainder)
414                .ok_or_else(|| malformed("AFF4 page offset overflow"))?,
415        };
416        while let Some(page_end) = page_start.checked_add(page_bytes) {
417            if page_end > request_end {
418                break;
419            }
420            let begin = usize::try_from(page_start - offset)
421                .map_err(|_| malformed("AFF4 page offset overflow"))?;
422            let page = Arc::new(buffer[begin..begin + LOGICAL_READ_AHEAD_BYTES].to_vec());
423            let key = self.disk_page_key(primary, id, page_start)?;
424            let mut cache = self.volumes[primary]
425                .decoded_cache
426                .lock()
427                .map_err(|_| malformed("AFF4 decoded cache lock poisoned"))?;
428            cache.record_read_ahead_prefetch();
429            cache.insert(key, page);
430            page_start = page_end;
431        }
432        Ok(())
433    }
434
435    fn read_disk_at_impl(
436        &mut self,
437        primary: usize,
438        mapped: bool,
439        id: &str,
440        buffer: &mut [u8],
441        offset: u64,
442        use_decoded_cache: bool,
443    ) -> Result<usize> {
444        if !mapped {
445            self.clear_previous_cache(primary);
446            return self.volumes[primary].read_at_impl(id, buffer, offset, use_decoded_cache);
447        }
448        let (map, size) = self.image_map_in(primary, id)?;
449        if offset >= size {
450            return Ok(0);
451        }
452        let length = (size - offset).min(buffer.len() as u64) as usize;
453        let mut done = 0;
454        while done < length {
455            let position = offset + done as u64;
456            let next = map.ranges.partition_point(|r| r.end <= position);
457            let (target, start, end) = match map.ranges.get(next) {
458                Some(range) if range.start <= position => (
459                    range.target.as_str(),
460                    range.offset + position - range.start,
461                    range.end,
462                ),
463                next => (map.gap.as_str(), position, next.map_or(size, |r| r.start)),
464            };
465            let take = (end - position).min((length - done) as u64) as usize;
466            let owner = if is_symbolic(target) {
467                0
468            } else {
469                *self
470                    .owners
471                    .get(target)
472                    .ok_or_else(|| malformed("missing target volume"))?
473            };
474            self.clear_previous_cache(owner);
475            self.volumes[owner].read_inner(
476                target,
477                &mut buffer[done..done + take],
478                start,
479                &mut Vec::new(),
480                use_decoded_cache,
481            )?;
482            done += take;
483        }
484        Ok(length)
485    }
486
487    fn clear_previous_cache(&mut self, owner: usize) {
488        if self.cached_owner != Some(owner) {
489            if let Some(previous) = self.cached_owner {
490                self.volumes[previous].cache = None;
491                self.volumes[previous].index_cache = None;
492                self.volumes[previous].maps.clear();
493            }
494            self.cached_owner = Some(owner);
495        }
496    }
497
498    /// Computes whole-image SHA256 across companions. This does not certify the
499    /// per-volume metadata or striped composite hashes. Optional external digest
500    /// must cover the assembled image, not information.turtle or a ZIP file.
501    pub fn verify_image(
502        &mut self,
503        id: &str,
504        expected_sha256: Option<&str>,
505        progress: impl FnMut(u64, u64) -> ControlFlow<()>,
506    ) -> Result<SetDigest> {
507        self.scan_image(id, expected_sha256, false, progress)
508            .map(|(digest, _)| digest)
509    }
510
511    fn scan_image(
512        &mut self,
513        id: &str,
514        expected_sha256: Option<&str>,
515        all_hashes: bool,
516        mut progress: impl FnMut(u64, u64) -> ControlFlow<()>,
517    ) -> Result<(SetDigest, Option<Verification>)> {
518        if expected_sha256
519            .is_some_and(|s| s.len() != 64 || !s.bytes().all(|b| b.is_ascii_hexdigit()))
520        {
521            return Err(malformed("invalid external SHA256"));
522        }
523        let size = self.size(id)?;
524        if size > self.volumes[0].limits.verification_bytes {
525            return Err(malformed("verification byte limit exceeded"));
526        }
527        let mut buffer = vec![0; 1024 * 1024];
528        let mut offset = 0;
529        let mut hash = Sha256::new();
530        let mut extra = all_hashes.then(|| {
531            (
532                Md5::new(),
533                Sha1::new(),
534                sha2::Sha512::new(),
535                blake2::Blake2b512::new(),
536            )
537        });
538        loop {
539            if progress(offset, size).is_break() {
540                return Err(Error::Aborted);
541            }
542            if offset == size {
543                break;
544            }
545            let read = self.read_disk_at_impl(0, true, id, &mut buffer, offset, false)?;
546            if read == 0 {
547                return Err(malformed("truncated volume set"));
548            }
549            hash.update(&buffer[..read]);
550            if let Some((md5, sha1, sha512, blake2b)) = &mut extra {
551                md5.update(&buffer[..read]);
552                sha1.update(&buffer[..read]);
553                sha512.update(&buffer[..read]);
554                blake2b.update(&buffer[..read]);
555            }
556            offset += read as u64;
557        }
558        let sha256 = hex(&hash.finalize());
559        let verification = extra.map(|(md5, sha1, sha512, blake2b)| Verification {
560            bytes_verified: size,
561            md5: hex(&md5.finalize()),
562            sha1: hex(&sha1.finalize()),
563            sha256: sha256.clone(),
564            sha512: hex(&sha512.finalize()),
565            blake2b: hex(&blake2b.finalize()),
566            references_match: None,
567            unsupported_hashes: Vec::new(),
568        });
569        Ok((
570            SetDigest {
571                image: id.into(),
572                bytes: size,
573                external_match: expected_sha256.map(|s| s.eq_ignore_ascii_case(&sha256)),
574                sha256,
575                sources: self.sources(),
576            },
577            verification,
578        ))
579    }
580}
581
582pub(super) fn is_symbolic(id: &str) -> bool {
583    id == format!("{NS}Zero")
584        || id.starts_with(&format!("{NS}SymbolicStream"))
585        || id == format!("{NS}UnknownData")
586        || id == format!("{NS}UnreadableData")
587}
588
589fn map_interval(map: &Map, position: u64, size: u64) -> (&str, u64, u64) {
590    let next = map.ranges.partition_point(|range| range.end <= position);
591    match map.ranges.get(next) {
592        Some(range) if range.start <= position => (
593            &range.target,
594            range.offset + position - range.start,
595            range.end,
596        ),
597        next => (&map.gap, position, next.map_or(size, |range| range.start)),
598    }
599}