Skip to main content

aff4_image/
collection_budget.rs

1//! Incremental collection accounting; finalized ZIP verification remains required.
2use super::*;
3
4/// Collection discovery bounds and the reader budgets used for staged verification.
5#[derive(Debug, Clone, serde::Serialize)]
6pub struct CollectionLimits {
7    /// Reader budgets; metadata/triples and ZIP entries are checked incrementally.
8    pub reader: crate::Limits,
9    /// Maximum discovered entries, including root, excluded and skipped entries.
10    pub entries: usize,
11    /// Maximum source depth; the root has depth zero.
12    pub depth: usize,
13}
14
15impl Default for CollectionLimits {
16    fn default() -> Self {
17        Self {
18            reader: crate::Limits::default(),
19            entries: 100_000,
20            depth: 127,
21        }
22    }
23}
24
25pub(super) fn check(resource: &'static str, required: u64, limit: u64) -> Result<()> {
26    if required > limit {
27        return Err(Error::ResourceLimit {
28            resource,
29            required,
30            limit,
31        });
32    }
33    Ok(())
34}
35
36pub(super) struct Budget<'a> {
37    limits: &'a CollectionLimits,
38    offset: usize,
39    roots: usize,
40    metadata: u64,
41    triples: u64,
42    bytes: u64,
43    discovered: usize,
44}
45
46impl<'a> Budget<'a> {
47    pub(super) fn new(writer: &Writer, limits: &'a CollectionLimits) -> Result<Self> {
48        let mut budget = Self {
49            limits,
50            offset: 0,
51            roots: 0,
52            metadata: 0,
53            triples: 0,
54            bytes: 0,
55            discovered: 0,
56        };
57        budget.observe(writer)?;
58        // Include deferred folder records and relationships already in the writer.
59        for (id, (properties, children)) in &writer.folders {
60            budget.fragment(&format!("<{id}> {properties} .\n"))?;
61            for child in children {
62                budget.charge(format!("; a:child <{child}>").len() as u64, 1)?;
63            }
64        }
65        // Reserve the final collection record, including worst-case decimal counts.
66        budget.fragment(&super::collect::task_metadata(&CollectionReport {
67            files: u64::MAX,
68            bytes: u64::MAX,
69            ..Default::default()
70        }))?;
71        for stream in &writer.streams {
72            budget.bytes = budget
73                .bytes
74                .checked_add(stream.size)
75                .ok_or_else(|| malformed("collection byte count overflow"))?;
76        }
77        budget.file(writer, None)?;
78        Ok(budget)
79    }
80
81    pub(super) fn discover(&mut self, depth: usize) -> Result<()> {
82        check("collection_depth", depth as u64, self.limits.depth as u64)?;
83        self.discovered = self
84            .discovered
85            .checked_add(1)
86            .ok_or_else(|| malformed("collection entry count overflow"))?;
87        check(
88            "collection_entries",
89            self.discovered as u64,
90            self.limits.entries as u64,
91        )
92    }
93
94    fn charge(&mut self, bytes: u64, triples: u64) -> Result<()> {
95        self.metadata = self
96            .metadata
97            .checked_add(bytes)
98            .ok_or_else(|| malformed("metadata byte count overflow"))?;
99        self.triples = self
100            .triples
101            .checked_add(triples)
102            .ok_or_else(|| malformed("metadata triple count overflow"))?;
103        check(
104            "metadata_bytes",
105            self.metadata,
106            self.limits.reader.metadata_bytes,
107        )?;
108        check("triples", self.triples, self.limits.reader.triples as u64)
109    }
110
111    fn fragment(&mut self, text: &str) -> Result<()> {
112        self.charge(text.len() as u64, 0)?;
113        let prefixes = b"@prefix a: <http://aff4.org/Schema#> .\n@prefix l: <https://aff4.org/Schema/2022/#> .\n";
114        for triple in
115            oxttl::TurtleParser::new().for_reader(prefixes.as_slice().chain(text.as_bytes()))
116        {
117            triple.map_err(|error| malformed(format!("writer metadata: {error}")))?;
118            self.charge(0, 1)?;
119        }
120        Ok(())
121    }
122
123    fn observe(&mut self, writer: &Writer) -> Result<()> {
124        self.fragment(&writer.metadata[self.offset..])?;
125        self.offset = writer.metadata.len();
126        if self.roots == 0 && !writer.roots.is_empty() {
127            // UUID length is fixed; these are the exact final header byte count.
128            let header = format!(
129                "<{}> a a:LogicalAcquisitionTask; l:pathSeparator {} .\n",
130                identifier(),
131                oxrdf::Literal::new_simple_literal(writer.path_separator.to_string())
132            );
133            self.fragment(&header)?;
134        }
135        for root in &writer.roots[self.roots..] {
136            self.charge(format!("; a:filesystemRoot <{root}>").len() as u64, 1)?;
137        }
138        self.roots = writer.roots.len();
139        Ok(())
140    }
141
142    pub(super) fn added(
143        &mut self,
144        writer: &Writer,
145        id: &str,
146        folder: bool,
147        parent: Option<&str>,
148    ) -> Result<()> {
149        self.observe(writer)?;
150        if folder {
151            let properties = &writer.folders[id].0;
152            self.fragment(&format!("<{id}> {properties} .\n"))?;
153        }
154        if parent.is_some() {
155            self.charge(format!("; a:child <{id}>").len() as u64, 1)?;
156        }
157        Ok(())
158    }
159
160    pub(super) fn issue(&mut self, issue: &CollectionIssue) -> Result<()> {
161        self.charge(super::collect::issue_metadata(issue).len() as u64, 1)
162    }
163
164    pub(super) fn file(&mut self, writer: &Writer, size: Option<u64>) -> Result<()> {
165        let mut entries = writer
166            .archive_entries
167            .checked_add(2)
168            .ok_or_else(|| malformed("archive entry count overflow"))?; // Final RDF and digest sidecar.
169        if let Some(size) = size {
170            let members = if size > writer.logical_zip_threshold {
171                size.div_ceil(u64::from(writer.options.chunk_bytes))
172                    .div_ceil(u64::from(writer.options.chunks_per_bevy))
173                    .checked_mul(4)
174                    .ok_or_else(|| malformed("archive entry count overflow"))?
175            } else {
176                1
177            };
178            entries = entries
179                .checked_add(members)
180                .ok_or_else(|| malformed("archive entry count overflow"))?;
181            self.bytes = self
182                .bytes
183                .checked_add(size)
184                .ok_or_else(|| malformed("collection byte count overflow"))?;
185        }
186        check(
187            "archive_entries",
188            entries,
189            self.limits.reader.archive_entries as u64,
190        )?;
191        // This is a lower bound: structural/block verification can traverse bytes again.
192        check(
193            "verification_bytes",
194            self.bytes,
195            self.limits.reader.verification_bytes,
196        )
197    }
198}
199
200#[cfg(test)]
201mod tests {
202    use super::*;
203
204    #[test]
205    fn accounts_for_deferred_hierarchy_and_collection_issues() {
206        let dir = tempfile::tempdir().unwrap();
207        let source = dir.path().join("source");
208        fs::create_dir_all(source.join("folder")).unwrap();
209        fs::write(source.join("folder/file"), b"abc").unwrap();
210        fs::write(source.join("skipped"), []).unwrap();
211        let mut writer = Writer::create(
212            dir.path().join("case.aff4"),
213            Profile::Logical,
214            WriteOptions::default(),
215        )
216        .unwrap();
217        writer
218            .add_case_metadata(&CaseMetadata {
219                notes: "quotes \" and slash \\".into(),
220                ..Default::default()
221            })
222            .unwrap();
223        writer
224            .add_directory_tree(
225                &source,
226                &CollectionOptions {
227                    exclude: vec!["skipped".into()],
228                    ..Default::default()
229                },
230                |_, _, _| ControlFlow::Continue(()),
231            )
232            .unwrap();
233        let limits = CollectionLimits::default();
234        let budget = Budget::new(&writer, &limits).unwrap();
235        let reserved = super::super::collect::task_metadata(&CollectionReport {
236            files: u64::MAX,
237            bytes: u64::MAX,
238            ..Default::default()
239        });
240        writer.finish_logical_metadata();
241        assert_eq!(
242            budget.metadata - reserved.len() as u64,
243            writer.metadata.len() as u64
244        );
245        let triples = oxttl::TurtleParser::new()
246            .for_reader(writer.metadata.as_bytes())
247            .inspect(|item| assert!(item.is_ok()))
248            .count();
249        assert_eq!(budget.triples - 4, triples as u64);
250    }
251}