Skip to main content

aff4_image/
collect.rs

1//! Local logical acquisition with explicit coverage and source-change checks.
2use super::*;
3use std::time::{SystemTime, UNIX_EPOCH};
4
5/// Local collection policy. Collection is not an atomic filesystem snapshot.
6#[derive(Debug, Clone, Default, serde::Serialize)]
7pub struct CollectionOptions {
8    /// Paths relative to the collection root to exclude, including descendants.
9    pub exclude: Vec<PathBuf>,
10    /// Publish a partial collection after discovery errors, recording every issue.
11    /// Read errors and source mutation always poison the writer.
12    pub allow_partial: bool,
13}
14
15/// One excluded, unsupported, or inaccessible source entry.
16#[derive(Debug, Clone, serde::Serialize)]
17pub struct CollectionIssue {
18    /// Original path; used only for reporting, never as an output destination.
19    pub path: PathBuf,
20    /// Why this entry was not collected.
21    pub reason: String,
22}
23
24/// Coverage of a local directory acquisition.
25#[derive(Debug, Clone, Default, serde::Serialize)]
26pub struct CollectionReport {
27    /// Successfully collected regular files.
28    pub files: u64,
29    /// Recorded folders, including empty ones.
30    pub folders: u64,
31    /// Original file bytes accepted.
32    pub bytes: u64,
33    /// Exclusions and skipped entries; an empty list is not snapshot certification.
34    pub issues: Vec<CollectionIssue>,
35}
36
37pub(super) fn issue_metadata(issue: &CollectionIssue) -> String {
38    format!(
39        "; a:collectionIssue {}",
40        oxrdf::Literal::new_simple_literal(format!("{}: {}", issue.path.display(), issue.reason))
41    )
42}
43
44pub(super) fn task_metadata(report: &CollectionReport) -> String {
45    let task = identifier();
46    let notes = oxrdf::Literal::new_simple_literal(
47        "Portable local collection; links not followed; cross-file snapshot consistency, automatic ADS/xattr/ACL capture not provided",
48    );
49    let mut text = format!(
50        "<{task}> a a:LogicalAcquisitionTask; a:notes {notes}; a:filesCollected {}; a:bytesCollected {}",
51        report.files, report.bytes
52    );
53    for issue in &report.issues {
54        text.push_str(&issue_metadata(issue));
55    }
56    text.push_str(" .\n");
57    text
58}
59
60fn record_issue(
61    report: &mut CollectionReport,
62    budget: &mut collection_budget::Budget<'_>,
63    issue: CollectionIssue,
64) -> Result<()> {
65    budget.issue(&issue)?;
66    report.issues.push(issue);
67    Ok(())
68}
69
70fn timestamp(value: std::io::Result<SystemTime>) -> Option<i128> {
71    value
72        .ok()
73        .map(|time| match time.duration_since(UNIX_EPOCH) {
74            Ok(n) => n.as_nanos() as i128,
75            Err(n) => -(n.duration().as_nanos() as i128),
76        })
77}
78
79fn source_metadata(
80    path: &Path,
81    meta: &fs::Metadata,
82    parent: Option<String>,
83) -> Result<LogicalMetadata> {
84    #[cfg(unix)]
85    let bytes = {
86        use std::os::unix::ffi::OsStrExt;
87        path.as_os_str().as_bytes().to_vec()
88    };
89    #[cfg(not(unix))]
90    let bytes = path
91        .to_str()
92        .ok_or_else(|| {
93            Error::Unsupported("source path is not Unicode; no lossy conversion is allowed".into())
94        })?
95        .as_bytes()
96        .to_vec();
97    let mut result = LogicalMetadata {
98        path: bytes,
99        parent,
100        created: timestamp(meta.created()),
101        modified: timestamp(meta.modified()),
102        accessed: timestamp(meta.accessed()),
103        ..LogicalMetadata::default()
104    };
105    #[cfg(unix)]
106    {
107        use std::os::unix::fs::MetadataExt;
108        result.mode = Some(meta.mode());
109        result.changed =
110            Some(i128::from(meta.ctime()) * 1_000_000_000 + i128::from(meta.ctime_nsec()));
111    }
112    // Windows does not expose a metadata-change timestamp through std::fs.
113    #[cfg(not(unix))]
114    {
115        result.mode = None;
116    }
117    Ok(result)
118}
119
120fn same_source(before: &fs::Metadata, after: &fs::Metadata) -> bool {
121    if before.len() != after.len() || before.modified().ok() != after.modified().ok() {
122        return false;
123    }
124    #[cfg(unix)]
125    {
126        use std::os::unix::fs::MetadataExt;
127        if before.ino() != after.ino()
128            || before.dev() != after.dev()
129            || before.ctime() != after.ctime()
130            || before.ctime_nsec() != after.ctime_nsec()
131        {
132            return false;
133        }
134    }
135    true
136}
137
138fn open_regular(path: &Path) -> Result<File> {
139    let mut options = fs::OpenOptions::new();
140    options.read(true);
141    #[cfg(unix)]
142    {
143        use std::os::unix::fs::OpenOptionsExt;
144        options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK);
145    }
146    #[cfg(windows)]
147    {
148        use std::os::windows::fs::OpenOptionsExt;
149        // FILE_FLAG_OPEN_REPARSE_POINT: open the link itself, not its target.
150        options.custom_flags(0x0020_0000);
151    }
152    let file = options.open(path)?;
153    let meta = file.metadata()?;
154    if !meta.is_file() {
155        return Err(Error::Unsupported("source is not a regular file".into()));
156    }
157    #[cfg(windows)]
158    {
159        use std::os::windows::fs::MetadataExt;
160        if meta.file_attributes() & 0x400 != 0 {
161            return Err(Error::Unsupported("reparse points are not followed".into()));
162        }
163    }
164    Ok(file)
165}
166
167impl Writer {
168    /// Collects a local directory tree using ordinary source permissions.
169    /// Symlinks/reparse points and special files are never followed. Exclusions
170    /// are explicit; skipped objects require `allow_partial`. Metadata records
171    /// the collection policy and omissions. Read errors, cancellation, or a
172    /// detected source change prevent publication. Use a stable source snapshot
173    /// when consistency across files is required; this is best-effort live reading.
174    /// ADS/xattrs/ACLs are not automatically captured by this portable collector.
175    pub fn add_directory_tree(
176        &mut self,
177        root: impl AsRef<Path>,
178        options: &CollectionOptions,
179        progress: impl FnMut(&Path, u64, u64) -> ControlFlow<()>,
180    ) -> Result<CollectionReport> {
181        self.add_directory_tree_with_limits(root, options, &CollectionLimits::default(), progress)
182    }
183
184    /// Collects with explicit discovery and incremental metadata/entry budgets.
185    /// Limits include excluded/skipped entries and deferred hierarchy/issue records.
186    /// Budget failures poison the writer even when partial collection is allowed.
187    /// Use the same reader limits with `finish_verified` for the full ZIP checks.
188    pub fn add_directory_tree_with_limits(
189        &mut self,
190        root: impl AsRef<Path>,
191        options: &CollectionOptions,
192        limits: &CollectionLimits,
193        mut progress: impl FnMut(&Path, u64, u64) -> ControlFlow<()>,
194    ) -> Result<CollectionReport> {
195        self.logical_parent(None)?;
196        if fs::symlink_metadata(root.as_ref())?
197            .file_type()
198            .is_symlink()
199        {
200            return Err(Error::Unsupported("root symlink is not followed".into()));
201        }
202        self.set_path_separator(std::path::MAIN_SEPARATOR)?;
203        let root = fs::canonicalize(root)?;
204        if !root.is_dir() {
205            return Err(malformed("collection source must be a directory"));
206        }
207        if self.path.starts_with(&root) {
208            return Err(malformed("output must be outside the collection tree"));
209        }
210        for excluded in &options.exclude {
211            if excluded.as_os_str().is_empty()
212                || excluded.components().any(|c| {
213                    !matches!(
214                        c,
215                        std::path::Component::Normal(_) | std::path::Component::CurDir
216                    )
217                })
218            {
219                return Err(malformed(
220                    "exclusions must be nonempty relative paths without parent traversal",
221                ));
222            }
223        }
224        let operation = (|| {
225            let mut budget = collection_budget::Budget::new(self, limits)?;
226            budget.discover(0)?;
227            let mut report = CollectionReport::default();
228            let mut pending = vec![(root.clone(), None, 0usize)];
229            while let Some((path, parent, depth)) = pending.pop() {
230                if progress(&path, 0, 0).is_break() {
231                    return Err(Error::Aborted);
232                }
233                let relative = path
234                    .strip_prefix(&root)
235                    .map_err(|_| malformed("source escaped root"))?;
236                if options.exclude.iter().any(|p| relative.starts_with(p)) {
237                    record_issue(
238                        &mut report,
239                        &mut budget,
240                        CollectionIssue {
241                            path,
242                            reason: "excluded by collection policy".into(),
243                        },
244                    )?;
245                    continue;
246                }
247                let discovered = match fs::symlink_metadata(&path) {
248                    Ok(meta) => meta,
249                    Err(e) if options.allow_partial => {
250                        record_issue(
251                            &mut report,
252                            &mut budget,
253                            CollectionIssue {
254                                path,
255                                reason: e.to_string(),
256                            },
257                        )?;
258                        continue;
259                    }
260                    Err(e) => return Err(e.into()),
261                };
262                let mut link = discovered.file_type().is_symlink();
263                #[cfg(windows)]
264                {
265                    use std::os::windows::fs::MetadataExt;
266                    link |= discovered.file_attributes() & 0x400 != 0;
267                }
268                #[cfg(not(windows))]
269                {
270                    link |= false;
271                }
272                if link || !discovered.is_file() && !discovered.is_dir() {
273                    let reason = "link/reparse point or special file not collected";
274                    if !options.allow_partial {
275                        return Err(Error::Unsupported(format!("{}: {reason}", path.display())));
276                    }
277                    record_issue(
278                        &mut report,
279                        &mut budget,
280                        CollectionIssue {
281                            path,
282                            reason: reason.into(),
283                        },
284                    )?;
285                    continue;
286                }
287                if discovered.is_dir() {
288                    let id =
289                        self.add_folder(&source_metadata(&path, &discovered, parent.clone())?)?;
290                    budget.added(self, &id, true, parent.as_deref())?;
291                    report.folders += 1;
292                    let children = (|| -> Result<Vec<PathBuf>> {
293                        let mut children = Vec::new();
294                        for entry in fs::read_dir(&path)? {
295                            if progress(&path, 0, 0).is_break() {
296                                return Err(Error::Aborted);
297                            }
298                            let entry = entry?;
299                            budget.discover(depth + 1)?;
300                            children.push(entry.path());
301                        }
302                        Ok(children)
303                    })();
304                    match children {
305                        Ok(mut children) => {
306                            children.sort();
307                            pending.extend(
308                                children
309                                    .into_iter()
310                                    .rev()
311                                    .map(|p| (p, Some(id.clone()), depth + 1)),
312                            );
313                        }
314                        Err(Error::Io(e)) if options.allow_partial => record_issue(
315                            &mut report,
316                            &mut budget,
317                            CollectionIssue {
318                                path,
319                                reason: e.to_string(),
320                            },
321                        )?,
322                        Err(e) => return Err(e),
323                    }
324                } else {
325                    let mut file = open_regular(&path)?;
326                    let before = file.metadata()?;
327                    if !same_source(&discovered, &before) {
328                        return Err(malformed("source changed before opening"));
329                    }
330                    budget.file(self, Some(before.len()))?;
331                    let id = self.add_file_with_metadata(
332                        &source_metadata(&path, &before, parent.clone())?,
333                        before.len(),
334                        &mut file,
335                        |done, size| progress(&path, done, size),
336                    )?;
337                    budget.added(self, &id, false, parent.as_deref())?;
338                    let after = file.metadata()?;
339                    if !same_source(&before, &after) {
340                        return Err(malformed(format!(
341                            "source changed during collection: {}",
342                            path.display()
343                        )));
344                    }
345                    report.files += 1;
346                    report.bytes = report
347                        .bytes
348                        .checked_add(before.len())
349                        .ok_or_else(|| malformed("collection byte count overflow"))?;
350                }
351            }
352            self.metadata.push_str(&task_metadata(&report));
353            Ok(report)
354        })();
355        if operation.is_err() {
356            self.poisoned = true;
357        }
358        operation
359    }
360}