Skip to main content

aff4_image/
scan.rs

1//! Streaming metadata inventory without retaining the RDF graph.
2use super::*;
3
4/// Work performed by a streaming metadata scan. Triples include repeated subjects.
5#[derive(Debug, Clone, Default, serde::Serialize)]
6pub struct MetadataScan {
7    /// Total triples delivered, including imported stores.
8    pub triples: usize,
9    /// Number of metadata ZIP members parsed.
10    pub stores: usize,
11    /// Uncompressed bytes of those members.
12    pub bytes: u64,
13}
14
15impl Container {
16    /// Streams primary and imported RDF statements without constructing a graph.
17    /// Subject order is arbitrary; callers must not assume adjacent statements
18    /// describe a complete object. The callback receives source-member provenance.
19    /// This is an inventory operation, not content or metadata verification.
20    /// ZIP directory and entry limits are checked before ZIP parsing.
21    pub fn scan_metadata(
22        path: impl AsRef<Path>,
23        limits: Limits,
24        mut visit: impl FnMut(&str, &str, &Property) -> ControlFlow<()>,
25    ) -> Result<MetadataScan> {
26        let (mut archive, _) = archive::open(path.as_ref(), &limits)?;
27        let mut names = BTreeSet::new();
28        for name in archive.file_names() {
29            if !names.insert(name.to_owned()) {
30                return Err(malformed("duplicate ZIP member name"));
31            }
32        }
33        let description = if archive.index_for_name("container.description").is_some() {
34            String::from_utf8(member(&mut archive, "container.description", 4096)?)
35                .map_err(|_| malformed("volume UTF8"))?
36        } else {
37            std::str::from_utf8(archive.comment())
38                .map_err(|_| malformed("volume UTF8"))?
39                .to_owned()
40        };
41        let volume = description.trim_end_matches('\0').trim();
42        if !volume.starts_with("aff4://") {
43            return Err(malformed("invalid AFF4 volume"));
44        }
45        let mut pending = vec!["information.turtle".to_owned()];
46        let mut seen = BTreeSet::new();
47        let mut result = MetadataScan::default();
48        while let Some(name) = pending.pop() {
49            if !seen.insert(name.clone()) {
50                continue;
51            }
52            let file = archive.by_name(&name)?;
53            let remaining = limits.metadata_bytes.saturating_sub(result.bytes);
54            if file.size() > remaining {
55                return Err(malformed("metadata byte limit exceeded"));
56            }
57            result.bytes += file.size();
58            result.stores += 1;
59            for triple in TurtleParser::new().for_reader(file.take(remaining.saturating_add(1))) {
60                if result.triples >= limits.triples {
61                    return Err(malformed("RDF triple limit exceeded"));
62                }
63                let triple = triple.map_err(|e| malformed(e.to_string()))?;
64                result.triples += 1;
65                let subject = match triple.subject {
66                    NamedOrBlankNode::NamedNode(n) => n.into_string(),
67                    NamedOrBlankNode::BlankNode(n) => format!("_:{n}"),
68                };
69                let (value, datatype, language) = match triple.object {
70                    Term::NamedNode(n) => (n.into_string(), None, None),
71                    Term::BlankNode(n) => (format!("_:{n}"), None, None),
72                    Term::Literal(l) => (
73                        l.value().to_owned(),
74                        Some(l.datatype().as_str().to_owned()),
75                        l.language().map(str::to_owned),
76                    ),
77                };
78                let property = Property {
79                    predicate: triple.predicate.into_string(),
80                    value,
81                    datatype,
82                    language,
83                };
84                if is_property(&property.predicate, "imports") {
85                    if property.datatype.is_some() {
86                        return Err(malformed("metadata import must be an IRI"));
87                    }
88                    let prefix = format!("{volume}/");
89                    let imported = property
90                        .value
91                        .strip_prefix(&prefix)
92                        .ok_or_else(|| Error::Unsupported("nonlocal metadata import".into()))?;
93                    pending.push(imported.to_owned());
94                }
95                if visit(&name, &subject, &property).is_break() {
96                    return Err(Error::Aborted);
97                }
98            }
99        }
100        Ok(result)
101    }
102}