aff4-image 0.2.0

Experimental Rust reader, verifier, and writer for AFF4 evidence containers
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
//! Metadata-based physical-image discovery without filename conventions.
use super::*;
use crate::DiskImageInfo;
use std::io::{self, Read, Seek, SeekFrom};
use std::path::PathBuf;
use std::sync::Mutex;

/// A container required by a discovered physical image.
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub struct BackingVolume {
    /// AFF4 volume identifier, checked again on reopen.
    pub volume_id: String,
    /// Container path supplied by the caller.
    pub path: PathBuf,
}

/// Stable identity and backing files for one decoded physical disk.
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub struct PhysicalDisk {
    /// Physical image resource and geometry.
    pub image: DiskImageInfo,
    /// Metadata-connected containers, excluding unrelated discovery candidates.
    pub volumes: Vec<BackingVolume>,
}

impl PhysicalDisk {
    /// Reopens exactly the recorded files and checks their identities and geometry.
    /// This does not hash the payload; backing files must remain unchanged.
    pub fn reopen(&self) -> Result<PhysicalDiskReader> {
        let paths: Vec<_> = self.volumes.iter().map(|v| v.path.clone()).collect();
        let set = DiskImageSet::discover(&paths, &[])?;
        set.into_readers()
            .into_iter()
            .find(|reader| reader.info() == self)
            .ok_or_else(|| malformed("AFF4 image or backing volume identity changed"))
    }
}

struct Inventory {
    container: Container,
    path: PathBuf,
    owned: BTreeSet<String>,
    needs: BTreeSet<String>,
    disks: Vec<(DiskImageInfo, bool)>,
}

fn symbolic(id: &str) -> bool {
    volume_set::is_symbolic(id)
}

fn dependencies(
    container: &mut Container,
    id: &str,
    stack: &mut Vec<String>,
    needs: &mut BTreeSet<String>,
) -> Result<()> {
    if symbolic(id) {
        return Ok(());
    }
    if container.inline_data(id)?.is_some() {
        return Ok(());
    }
    if stack.len() >= 32 || stack.iter().any(|previous| previous == id) {
        return Err(malformed("cyclic or excessive disk dependency depth"));
    }
    stack.push(id.to_owned());
    if let Some(target) = container.value(id, "dataStream")? {
        dependencies(container, &target, stack, needs)?;
    } else if container.has_type(id, "Map") {
        container.load_map(id)?;
        let map = container.maps[id].clone();
        let mut targets: BTreeSet<_> = map
            .ranges
            .iter()
            .map(|range| range.target.clone())
            .collect();
        targets.insert(map.gap.clone());
        for target in targets {
            dependencies(container, &target, stack, needs)?;
        }
    } else {
        let path = container.path(id)?;
        let local = container.archive.index_for_name(&path).is_some()
            || container
                .archive
                .index_for_name(&format!("{path}/00000000"))
                .is_some()
            || (container.value(id, "stored")?.as_deref() == Some(&container.volume)
                && container.number(id, "size").ok() == Some(0));
        if !local {
            needs.insert(id.to_owned());
        }
    }
    stack.pop();
    Ok(())
}

impl Inventory {
    fn new(mut container: Container, path: PathBuf) -> Result<Self> {
        let mut owned = BTreeSet::from([container.volume.clone()]);
        let mut needs = BTreeSet::new();
        for id in container.graph.keys() {
            if container.has_type(id, "ImageStream") {
                // ZipVolume::stored may be an original filename, and case
                // metadata is not a disk dependency. Resolve storage owners.
                if let Some(stored) = container.value(id, "stored")?
                    && stored != container.volume
                {
                    needs.insert(stored);
                }
                let member = format!("{}/00000000", container.path(id)?);
                if container.archive.index_for_name(&member).is_some()
                    || (container.number(id, "size").ok() == Some(0)
                        && container.value(id, "stored")?.as_deref() == Some(&container.volume))
                {
                    owned.insert(id.clone());
                }
            }
        }
        let mut disks = Vec::new();
        for disk in container.disk_images()? {
            let mut dependencies_found = BTreeSet::new();
            dependencies(
                &mut container,
                &disk.resource_id,
                &mut Vec::new(),
                &mut dependencies_found,
            )?;
            let mapped = !dependencies_found.is_empty();
            needs.extend(dependencies_found);
            disks.push((disk, mapped));
        }
        container.maps.clear();
        Ok(Self {
            container,
            path,
            owned,
            needs,
            disks,
        })
    }
}

/// Physical disks discovered from explicit inputs and optional nearby containers.
/// Membership follows volume identifiers and map references, never filenames.
pub struct DiskImageSet {
    readers: Vec<PhysicalDiskReader>,
}

impl DiskImageSet {
    /// Discovers every physical disk connected to an input, including when the
    /// input is a companion. Unrelated candidates are not returned as evidence.
    /// Candidates must be local paths; this API never follows network references.
    pub fn discover(inputs: &[PathBuf], candidates: &[PathBuf]) -> Result<Self> {
        Self::discover_with_limits(inputs, candidates, Limits::default())
    }

    /// Discovery inspects at most 128 distinct paths, with aggregate metadata,
    /// directory and triple budgets. Invalid optional candidates are ignored;
    /// unresolved dependencies of selected evidence remain errors.
    pub fn discover_with_limits(
        inputs: &[PathBuf],
        candidates: &[PathBuf],
        limits: Limits,
    ) -> Result<Self> {
        if inputs.is_empty() {
            return Err(malformed("no AFF4 inputs"));
        }
        let mut paths = Vec::new();
        for path in inputs.iter().chain(candidates) {
            if !paths.contains(path) {
                paths.push(path.clone());
            }
        }
        if paths.len() > 128 {
            return Err(malformed("AFF4 discovery exceeds 128 container paths"));
        }
        let mut remaining = limits.clone();
        let mut inventory = Vec::new();
        let mut selected = BTreeSet::new();
        for path in paths {
            let explicit = inputs.contains(&path);
            let mut container = match Container::open_with_limits(&path, remaining.clone()) {
                Ok(container) => container,
                Err(error) if explicit => return Err(error),
                Err(_) => continue,
            };
            remaining.metadata_bytes -= container.usage.metadata_bytes;
            remaining.triples -= container.usage.triples;
            remaining.directory_bytes -= container.usage.directory_bytes;
            remaining.archive_entries -= container.usage.entries;
            container.limits = limits.clone();
            match Inventory::new(container, path) {
                Ok(item) => {
                    if explicit {
                        selected.insert(inventory.len());
                    }
                    inventory.push(item);
                }
                Err(error) if explicit => return Err(error),
                Err(_) => continue,
            }
        }
        let mut providers: BTreeMap<String, Vec<usize>> = BTreeMap::new();
        for (index, item) in inventory.iter().enumerate() {
            for id in &item.owned {
                providers.entry(id.clone()).or_default().push(index);
            }
        }
        let mut edges = vec![BTreeSet::new(); inventory.len()];
        for (index, item) in inventory.iter().enumerate() {
            for need in &item.needs {
                for &owner in providers.get(need).into_iter().flatten() {
                    edges[index].insert(owner);
                    edges[owner].insert(index);
                }
            }
        }
        loop {
            let previous = selected.len();
            for index in selected.clone() {
                selected.extend(&edges[index]);
            }
            if selected.len() == previous {
                break;
            }
        }
        for &index in &selected {
            for owned in &inventory[index].owned {
                if providers[owned].len() != 1 {
                    return Err(malformed(format!("ambiguous AFF4 owner for {owned}")));
                }
            }
            for need in &inventory[index].needs {
                match providers.get(need).map(Vec::len).unwrap_or(0) {
                    1 => (),
                    0 => return Err(malformed(format!("missing AFF4 companion for {need}"))),
                    _ => return Err(malformed(format!("ambiguous AFF4 companion for {need}"))),
                }
            }
        }
        let mut descriptions = Vec::new();
        for (primary, &index) in selected.iter().enumerate() {
            let mut component = BTreeSet::from([index]);
            loop {
                let previous = component.len();
                for member in component.clone() {
                    component.extend(&edges[member]);
                }
                if previous == component.len() {
                    break;
                }
            }
            if component
                .iter()
                .all(|&member| inventory[member].disks.is_empty())
            {
                return Err(Error::Unsupported(format!(
                    "no physical DiskImage found for input {}",
                    inventory[index].path.display()
                )));
            }
            let mut volumes: Vec<_> = component
                .iter()
                .map(|&member| BackingVolume {
                    volume_id: inventory[member].container.volume.clone(),
                    path: inventory[member].path.clone(),
                })
                .collect();
            volumes.sort_by(|left, right| left.volume_id.cmp(&right.volume_id));
            for (image, mapped) in &inventory[index].disks {
                if descriptions.len() >= 128 {
                    return Err(malformed("more than 128 physical disk images"));
                }
                descriptions.push((
                    primary,
                    *mapped,
                    PhysicalDisk {
                        image: image.clone(),
                        volumes: volumes.clone(),
                    },
                ));
            }
        }
        if descriptions.is_empty() {
            return Err(Error::Unsupported(
                "no physical DiskImage resources found".into(),
            ));
        }
        let mut containers = Vec::new();
        let mut paths = Vec::new();
        for (index, item) in inventory.into_iter().enumerate() {
            if selected.contains(&index) {
                containers.push(item.container);
                paths.push(item.path);
            }
        }
        let mut backing = VolumeSet::from_containers(containers, paths)?;
        let mut unique: BTreeMap<String, (usize, bool, PhysicalDisk)> = BTreeMap::new();
        for (primary, mapped, info) in descriptions {
            if backing.disk_size(primary, mapped, &info.image.resource_id)?
                != info.image.logical_size
            {
                return Err(malformed("physical image length changed during discovery"));
            }
            if let Some((previous, _, original)) = unique.get(&info.image.resource_id) {
                if original.image.logical_size != info.image.logical_size
                    || original.image.block_size != info.image.block_size
                    || original.volumes != info.volumes
                    || !backing.equivalent_disk_maps(*previous, primary, &info.image.resource_id)?
                {
                    return Err(malformed("conflicting definitions of physical image"));
                }
                // Producers may replicate a disk with differently split Maps.
                // Choose a stable primary only after proving equivalent ranges.
                if original.image.volume_id < info.image.volume_id {
                    continue;
                }
            }
            unique.insert(info.image.resource_id.clone(), (primary, mapped, info));
        }
        let backing = Arc::new(Mutex::new(backing));
        Ok(Self {
            readers: unique
                .into_values()
                .map(|(primary, mapped, info)| PhysicalDiskReader {
                    backing: backing.clone(),
                    primary,
                    mapped,
                    info,
                    position: 0,
                })
                .collect(),
        })
    }

    /// Transfers all discovered disks to independent cursors with shared caches.
    pub fn into_readers(self) -> Vec<PhysicalDiskReader> {
        self.readers
    }
}

/// Seekable decoded disk, including data spread across companion containers.
pub struct PhysicalDiskReader {
    backing: Arc<Mutex<VolumeSet>>,
    primary: usize,
    mapped: bool,
    info: PhysicalDisk,
    position: u64,
}

impl PhysicalDiskReader {
    /// Identity, geometry and exact backing container paths.
    pub fn info(&self) -> &PhysicalDisk {
        &self.info
    }

    /// Returns cumulative payload-reader counters shared by this volume set.
    pub fn reader_statistics(&self) -> Result<ReaderStatistics> {
        Ok(self
            .backing
            .lock()
            .map_err(|_| malformed("AFF4 reader lock poisoned"))?
            .reader_statistics())
    }

    /// Returns automatic decoded payload-cache usage shared by this volume set.
    pub fn reader_cache_info(&self) -> Result<ReaderCacheInfo> {
        Ok(self
            .backing
            .lock()
            .map_err(|_| malformed("AFF4 reader lock poisoned"))?
            .reader_cache_info())
    }

    /// Reads decoded bytes without moving the cursor.
    pub fn read_at(&mut self, buffer: &mut [u8], offset: u64) -> Result<usize> {
        self.backing
            .lock()
            .map_err(|_| malformed("AFF4 reader lock poisoned"))?
            .read_disk_at(
                self.primary,
                self.mapped,
                &self.info.image.resource_id,
                buffer,
                offset,
            )
    }
}

impl Read for PhysicalDiskReader {
    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
        let count = self
            .read_at(buffer, self.position)
            .map_err(io::Error::other)?;
        self.position += count as u64;
        Ok(count)
    }
}

impl Seek for PhysicalDiskReader {
    fn seek(&mut self, from: SeekFrom) -> io::Result<u64> {
        let position = match from {
            SeekFrom::Start(offset) => Some(offset),
            SeekFrom::End(offset) => self.info.image.logical_size.checked_add_signed(offset),
            SeekFrom::Current(offset) => self.position.checked_add_signed(offset),
        }
        .ok_or_else(|| {
            io::Error::new(io::ErrorKind::InvalidInput, "AFF4 seek outside u64 range")
        })?;
        self.position = position;
        Ok(position)
    }
}